diff options
Diffstat (limited to 'lib')
298 files changed, 40200 insertions, 5628 deletions
diff --git a/lib/.gitignore b/lib/.gitignore index f2a39c9e5485..e5e217b8307b 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -1,8 +1,8 @@ -# -# Generated files -# -gen_crc32table -gen_crc64table -crc32table.h -crc64table.h -oid_registry_data.c +# SPDX-License-Identifier: GPL-2.0-only +/crc32table.h +/crc64table.h +/gen_crc32table +/gen_crc64table +/oid_registry_data.c +/test_fortify.log +/test_fortify/*.log diff --git a/lib/Kconfig b/lib/Kconfig index 6e790dc55c5b..5e7165e6a346 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -19,6 +19,9 @@ config RAID6_PQ_BENCHMARK Benchmark all available RAID6 PQ functions on init and choose the fastest one. +config LINEAR_RANGES + tristate + config PACKING bool "Generic bitfield packing and unpacking" default n @@ -47,12 +50,18 @@ config HAVE_ARCH_BITREVERSE This option enables the use of hardware bit-reversal instructions on architectures which support such operations. -config GENERIC_STRNCPY_FROM_USER +config ARCH_HAS_STRNCPY_FROM_USER bool -config GENERIC_STRNLEN_USER +config ARCH_HAS_STRNLEN_USER bool +config GENERIC_STRNCPY_FROM_USER + def_bool !ARCH_HAS_STRNCPY_FROM_USER + +config GENERIC_STRNLEN_USER + def_bool !ARCH_HAS_STRNLEN_USER + config GENERIC_NET_UTILS bool @@ -80,6 +89,9 @@ config ARCH_USE_CMPXCHG_LOCKREF config ARCH_HAS_FAST_MULTIPLIER bool +config ARCH_USE_SYM_ANNOTATIONS + bool + config INDIRECT_PIO bool "Access I/O in non-MMIO mode" depends on ARM64 @@ -96,6 +108,20 @@ config INDIRECT_PIO When in doubt, say N. +config INDIRECT_IOMEM + bool + help + This is selected by other options/architectures to provide the + emulated iomem accessors. + +config INDIRECT_IOMEM_FALLBACK + bool + depends on INDIRECT_IOMEM + help + If INDIRECT_IOMEM is selected, this enables falling back to plain + mmio accesses when the IO memory address is not a registered + emulated region. + config CRC_CCITT tristate "CRC-CCITT functions" help @@ -278,6 +304,13 @@ config ZLIB_DEFLATE tristate select BITREVERSE +config ZLIB_DFLTCC + def_bool y + depends on S390 + prompt "Enable s390x DEFLATE CONVERSION CALL support for kernel zlib" + help + Enable s390x hardware support for zlib in the kernel. + config LZO_COMPRESS tristate @@ -329,6 +362,10 @@ config DECOMPRESS_LZ4 select LZ4_DECOMPRESS tristate +config DECOMPRESS_ZSTD + select ZSTD_DECOMPRESS + tristate + # # Generic allocator support is selected if needed # @@ -420,7 +457,7 @@ config INTERVAL_TREE See: - Documentation/rbtree.txt + Documentation/core-api/rbtree.rst for more information. @@ -608,6 +645,9 @@ config ARCH_HAS_PMEM_API config MEMREGION bool +config ARCH_HAS_MEMREMAP_COMPAT_ALIGN + bool + # use memcpy to implement user copies for nommu architectures config UACCESS_MEMCPY bool @@ -615,7 +655,12 @@ config UACCESS_MEMCPY config ARCH_HAS_UACCESS_FLUSHCACHE bool -config ARCH_HAS_UACCESS_MCSAFE +# arch has a concept of a recoverable synchronous exception due to a +# memory-read error like x86 machine-check or ARM data-abort, and +# implements copy_mc_to_{user,kernel} to abort and report +# 'bytes-transferred' if that exception fires when accessing the source +# buffer. +config ARCH_HAS_COPY_MC bool # Temporary. Goes away when all archs are cleaned up @@ -626,6 +671,15 @@ config STACKDEPOT bool select STACKTRACE +config STACK_HASH_ORDER + int "stack depot hash size (12 => 4KB, 20 => 1024KB)" + range 12 20 + default 20 + depends on STACKDEPOT + help + Select the hash size as a power of 2 for the stackdepot hash table. + Choose a lower value to reduce the memory impact. + config SBITMAP bool @@ -635,9 +689,6 @@ config PARMAN config OBJAGG tristate "objagg" if COMPILE_TEST -config STRING_SELFTEST - tristate "Test string functions" - endmenu config GENERIC_IOREMAP @@ -660,3 +711,13 @@ config GENERIC_LIB_CMPDI2 config GENERIC_LIB_UCMPDI2 bool + +config GENERIC_LIB_DEVMEM_IS_ALLOWED + bool + +config PLDMFW + bool + default n + +config ASN1_ENCODER + tristate diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 5ffe144c9794..9ef7ce18b4f5 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -35,6 +35,17 @@ config PRINTK_CALLER no option to enable/disable at the kernel command line parameter or sysfs interface. +config STACKTRACE_BUILD_ID + bool "Show build ID information in stacktraces" + depends on PRINTK + help + Selecting this option adds build ID information for symbols in + stacktraces printed with the printk format '%p[SR]b'. + + This option is intended for distros where debuginfo is not easily + accessible but can be downloaded given the build ID of the vmlinux or + kernel module where the function is located. + config CONSOLE_LOGLEVEL_DEFAULT int "Default console loglevel (1-15)" range 1 15 @@ -98,7 +109,8 @@ config DYNAMIC_DEBUG bool "Enable dynamic printk() support" default n depends on PRINTK - depends on DEBUG_FS + depends on (DEBUG_FS || PROC_FS) + select DYNAMIC_DEBUG_CORE help Compiles debug level messages into the kernel, which would not @@ -116,8 +128,9 @@ config DYNAMIC_DEBUG Usage: Dynamic debugging is controlled via the 'dynamic_debug/control' file, - which is contained in the 'debugfs' filesystem. Thus, the debugfs - filesystem must first be mounted before making use of this feature. + which is contained in the 'debugfs' filesystem or procfs. + Thus, the debugfs or procfs filesystem must first be mounted before + making use of this feature. We refer the control file as: <debugfs>/dynamic_debug/control. This file contains a list of the debug statements that can be enabled. The format for each line of the file is: @@ -164,6 +177,17 @@ config DYNAMIC_DEBUG See Documentation/admin-guide/dynamic-debug-howto.rst for additional information. +config DYNAMIC_DEBUG_CORE + bool "Enable core function of dynamic debug support" + depends on PRINTK + depends on (DEBUG_FS || PROC_FS) + help + Enable core functional support of dynamic debug. It is useful + when you want to tie dynamic debug to your kernel modules with + DYNAMIC_DEBUG_MODULE defined for each of them, especially for + the case of embedded system where the kernel image size is + sensitive for people. + config SYMBOLIC_ERRNAME bool "Support symbolic error names in printf" default y if PRINTK @@ -199,9 +223,10 @@ config DEBUG_INFO If unsure, say N. +if DEBUG_INFO + config DEBUG_INFO_REDUCED bool "Reduce debugging information" - depends on DEBUG_INFO help If you say Y here gcc is instructed to generate less debugging information for structure types. This means that tools that @@ -212,9 +237,23 @@ config DEBUG_INFO_REDUCED DEBUG_INFO build and compile times are reduced too. Only works with newer gcc versions. +config DEBUG_INFO_COMPRESSED + bool "Compressed debugging information" + depends on $(cc-option,-gz=zlib) + depends on $(ld-option,--compress-debug-sections=zlib) + help + Compress the debug information using zlib. Requires GCC 5.0+ or Clang + 5.0+, binutils 2.26+, and zlib. + + Users of dpkg-deb via scripts/package/builddeb may find an increase in + size of their debug .deb packages with this config set, due to the + debug info being compressed with zlib, then the object files being + recompressed with a different compression scheme. But this is still + preferable to setting $KDEB_COMPRESS to "none" which would be even + larger. + config DEBUG_INFO_SPLIT bool "Produce split debuginfo in .dwo files" - depends on DEBUG_INFO depends on $(cc-option,-gsplit-dwarf) help Generate debug info into separate .dwo files. This significantly @@ -228,27 +267,71 @@ config DEBUG_INFO_SPLIT to know about the .dwo files and include them. Incompatible with older versions of ccache. +choice + prompt "DWARF version" + help + Which version of DWARF debug info to emit. + +config DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT + bool "Rely on the toolchain's implicit default DWARF version" + help + The implicit default version of DWARF debug info produced by a + toolchain changes over time. + + This can break consumers of the debug info that haven't upgraded to + support newer revisions, and prevent testing newer versions, but + those should be less common scenarios. + + If unsure, say Y. + config DEBUG_INFO_DWARF4 - bool "Generate dwarf4 debuginfo" - depends on DEBUG_INFO - depends on $(cc-option,-gdwarf-4) + bool "Generate DWARF Version 4 debuginfo" help - Generate dwarf4 debug info. This requires recent versions - of gcc and gdb. It makes the debug information larger. - But it significantly improves the success of resolving - variables in gdb on optimized code. + Generate DWARF v4 debug info. This requires gcc 4.5+ and gdb 7.0+. + + If you have consumers of DWARF debug info that are not ready for + newer revisions of DWARF, you may wish to choose this or have your + config select this. + +config DEBUG_INFO_DWARF5 + bool "Generate DWARF Version 5 debuginfo" + depends on !CC_IS_CLANG || (CC_IS_CLANG && (AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502))) + depends on !DEBUG_INFO_BTF + help + Generate DWARF v5 debug info. Requires binutils 2.35.2, gcc 5.0+ (gcc + 5.0+ accepts the -gdwarf-5 flag but only had partial support for some + draft features until 7.0), and gdb 8.0+. + + Changes to the structure of debug info in Version 5 allow for around + 15-18% savings in resulting image and debug info section sizes as + compared to DWARF Version 4. DWARF Version 5 standardizes previous + extensions such as accelerators for symbol indexing and the format + for fission (.dwo/.dwp) files. Users may not want to select this + config if they rely on tooling that has not yet been updated to + support DWARF Version 5. + +endchoice # "DWARF version" config DEBUG_INFO_BTF bool "Generate BTF typeinfo" - depends on DEBUG_INFO + depends on !DEBUG_INFO_SPLIT && !DEBUG_INFO_REDUCED + depends on !GCC_PLUGIN_RANDSTRUCT || COMPILE_TEST help Generate deduplicated BTF type information from DWARF debug info. Turning this on expects presence of pahole tool, which will convert DWARF type info into equivalent deduplicated BTF type info. +config PAHOLE_HAS_SPLIT_BTF + def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "119") + +config DEBUG_INFO_BTF_MODULES + def_bool y + depends on DEBUG_INFO_BTF && MODULES && PAHOLE_HAS_SPLIT_BTF + help + Generate compact split BTF type information for kernel modules. + config GDB_SCRIPTS bool "Provide GDB scripts for kernel debugging" - depends on DEBUG_INFO help This creates the required links to GDB helper scripts in the build directory. If you load vmlinux into gdb, the helper @@ -257,26 +340,19 @@ config GDB_SCRIPTS instance. See Documentation/dev-tools/gdb-kernel-debugging.rst for further details. -config ENABLE_MUST_CHECK - bool "Enable __must_check logic" - default y - help - Enable the __must_check logic in the kernel build. Disable this to - suppress the "warning: ignoring return value of 'foo', declared with - attribute warn_unused_result" messages. +endif # DEBUG_INFO config FRAME_WARN - int "Warn for stack frames larger than (needs gcc 4.4)" + int "Warn for stack frames larger than" range 0 8192 default 2048 if GCC_PLUGIN_LATENT_ENTROPY - default 1280 if (!64BIT && PARISC) + default 1536 if (!64BIT && (PARISC || XTENSA)) default 1024 if (!64BIT && !PARISC) default 2048 if 64BIT help Tell gcc to warn at build time for stack frames larger than this. Setting this too low will cause a lot of warnings. Setting it to 0 disables the warning. - Requires gcc 4.4 config STRIP_ASM_SYMS bool "Strip assembler-generated symbols during link" @@ -289,6 +365,7 @@ config STRIP_ASM_SYMS config READABLE_ASM bool "Generate readable assembler code" depends on DEBUG_KERNEL + depends on CC_IS_GCC help Disable some compiler optimizations that tend to generate human unreadable assembler output. This may make the kernel slightly slower, but it helps @@ -305,20 +382,9 @@ config HEADERS_INSTALL user-space program samples. It is also needed by some features such as uapi header sanity checks. -config OPTIMIZE_INLINING - def_bool y - help - This option determines if the kernel forces gcc to inline the functions - developers have marked 'inline'. Doing so takes away freedom from gcc to - do what it thinks is best, which is desirable for the gcc 3.x series of - compilers. The gcc 4.x series have a rewritten inlining algorithm and - enabling this option will generate a smaller kernel there. Hopefully - this algorithm is so good that allowing gcc 4.x and above to make the - decision will become the default in the future. Until then this option - is there to test gcc for this. - config DEBUG_SECTION_MISMATCH bool "Enable full Section mismatch analysis" + depends on CC_IS_GCC help The section mismatch analysis checks if there are illegal references from one section to another section. @@ -347,6 +413,17 @@ config SECTION_MISMATCH_WARN_ONLY If unsure, say Y. +config DEBUG_FORCE_FUNCTION_ALIGN_64B + bool "Force all function address 64B aligned" if EXPERT + help + There are cases that a commit from one domain changes the function + address alignment of other domains, and cause magic performance + bump (regression or improvement). Enable this option will help to + verify if the bump is caused by function alignment changes, while + it will slightly increase the kernel size and affect icache usage. + + It is mainly for debug and performance tuning use. + # # Select this config option from the architecture Kconfig, if it # is preferred to always offer frame pointers as a config @@ -379,6 +456,21 @@ config STACK_VALIDATION For more information, see tools/objtool/Documentation/stack-validation.txt. +config VMLINUX_VALIDATION + bool + depends on STACK_VALIDATION && DEBUG_ENTRY + default y + +config VMLINUX_MAP + bool "Generate vmlinux.map file when linking" + depends on EXPERT + help + Selecting this option will pass "-Map=vmlinux.map" to ld + when linking vmlinux. That file can be useful for verifying + and debugging magic section games, and for seeing which + pieces of code get eliminated with + CONFIG_LD_DEAD_CODE_DATA_ELIMINATION. + config DEBUG_FORCE_WEAK_PER_CPU bool "Force weak per-cpu definitions" depends on DEBUG_KERNEL @@ -431,6 +523,16 @@ config MAGIC_SYSRQ_SERIAL This option allows you to decide whether you want to enable the magic SysRq key. +config MAGIC_SYSRQ_SERIAL_SEQUENCE + string "Char sequence that enables magic SysRq over serial" + depends on MAGIC_SYSRQ_SERIAL + default "" + help + Specifies a sequence of characters that can follow BREAK to enable + SysRq on a serial console. + + If unsure, leave an empty string and the option will not be enabled. + config DEBUG_FS bool "Debug Filesystem" help @@ -443,9 +545,41 @@ config DEBUG_FS If unsure, say N. -source "lib/Kconfig.kgdb" +choice + prompt "Debugfs default access" + depends on DEBUG_FS + default DEBUG_FS_ALLOW_ALL + help + This selects the default access restrictions for debugfs. + It can be overridden with kernel command line option + debugfs=[on,no-mount,off]. The restrictions apply for API access + and filesystem registration. + +config DEBUG_FS_ALLOW_ALL + bool "Access normal" + help + No restrictions apply. Both API and filesystem registration + is on. This is the normal default operation. + +config DEBUG_FS_DISALLOW_MOUNT + bool "Do not register debugfs as filesystem" + help + The API is open but filesystem is not loaded. Clients can still do + their work and read with debug tools that do not need + debugfs filesystem. + +config DEBUG_FS_ALLOW_NONE + bool "No access" + help + Access is off. Clients get -PERM when trying to create nodes in + debugfs tree and debugfs is not registered as a filesystem. + Client can then back-off or continue without debugfs access. + +endchoice +source "lib/Kconfig.kgdb" source "lib/Kconfig.ubsan" +source "lib/Kconfig.kcsan" endmenu @@ -653,6 +787,12 @@ config SCHED_STACK_END_CHECK data corruption or a sporadic crash at a later stage once the region is examined. The runtime overhead introduced is minimal. +config ARCH_HAS_DEBUG_VM_PGTABLE + bool + help + An architecture should select this when it can successfully + build and run DEBUG_VM_PGTABLE. + config DEBUG_VM bool "Debug VM" depends on DEBUG_KERNEL @@ -688,6 +828,22 @@ config DEBUG_VM_PGFLAGS If unsure, say N. +config DEBUG_VM_PGTABLE + bool "Debug arch page table for semantics compliance" + depends on MMU + depends on ARCH_HAS_DEBUG_VM_PGTABLE + default y if DEBUG_VM + help + This option provides a debug method which can be used to test + architecture page table helper functions on various platforms in + verifying if they comply with expected generic MM semantics. This + will help architecture code in making sure that any changes or + new additions of these helpers still conform to expected + semantics of the generic MM. Platforms will have to opt in for + this through ARCH_HAS_DEBUG_VM_PGTABLE. + + If unsure, say N. + config ARCH_HAS_DEBUG_VIRTUAL bool @@ -721,7 +877,7 @@ config DEBUG_MEMORY_INIT config MEMORY_NOTIFIER_ERROR_INJECT tristate "Memory hotplug notifier error injection module" - depends on MEMORY_HOTPLUG_SPARSE && NOTIFIER_ERROR_INJECTION + depends on MEMORY_HOTPLUG && NOTIFIER_ERROR_INJECTION help This option provides the ability to inject artificial errors to memory hotplug notifier chain callbacks. It is controlled through @@ -753,9 +909,31 @@ config DEBUG_PER_CPU_MAPS Say N if unsure. +config DEBUG_KMAP_LOCAL + bool "Debug kmap_local temporary mappings" + depends on DEBUG_KERNEL && KMAP_LOCAL + help + This option enables additional error checking for the kmap_local + infrastructure. Disable for production use. + +config ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP + bool + +config DEBUG_KMAP_LOCAL_FORCE_MAP + bool "Enforce kmap_local temporary mappings" + depends on DEBUG_KERNEL && ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP + select KMAP_LOCAL + select DEBUG_KMAP_LOCAL + help + This option enforces temporary mappings through the kmap_local + mechanism for non-highmem pages and on non-highmem systems. + Disable this for production systems! + config DEBUG_HIGHMEM bool "Highmem debugging" depends on DEBUG_KERNEL && HIGHMEM + select DEBUG_KMAP_LOCAL_FORCE_MAP if ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP + select DEBUG_KMAP_LOCAL help This option enables additional error checking for high memory systems. Disable for production systems. @@ -766,7 +944,7 @@ config HAVE_DEBUG_STACKOVERFLOW config DEBUG_STACKOVERFLOW bool "Check for stack overflows" depends on DEBUG_KERNEL && HAVE_DEBUG_STACKOVERFLOW - ---help--- + help Say Y here if you want to check for overflows of kernel, IRQ and exception stacks (if your architecture uses them). This option will show detailed messages if free stack space drops @@ -782,6 +960,7 @@ config DEBUG_STACKOVERFLOW If in doubt, say "N". source "lib/Kconfig.kasan" +source "lib/Kconfig.kfence" endmenu # "Memory Debugging" @@ -789,10 +968,10 @@ config DEBUG_SHIRQ bool "Debug shared IRQ handlers" depends on DEBUG_KERNEL help - Enable this to generate a spurious interrupt as soon as a shared - interrupt handler is registered, and just before one is deregistered. - Drivers ought to be able to handle interrupts coming in at those - points; some don't and need to be caught. + Enable this to generate a spurious interrupt just before a shared + interrupt handler is deregistered (generating one when registering + is currently disabled). Drivers need to handle this correctly. Some + don't and need to be caught. menu "Debug Oops, Lockups and Hangs" @@ -819,7 +998,7 @@ config PANIC_TIMEOUT int "panic timeout" default 0 help - Set the timeout value (in seconds) until a reboot occurs when the + Set the timeout value (in seconds) until a reboot occurs when the kernel panics. If n = 0, then we wait forever. A timeout value n > 0 will wait n seconds before rebooting, while a timeout value n < 0 will reboot immediately. @@ -885,7 +1064,6 @@ config HARDLOCKUP_DETECTOR depends on HAVE_HARDLOCKUP_DETECTOR_PERF || HAVE_HARDLOCKUP_DETECTOR_ARCH select LOCKUP_DETECTOR select HARDLOCKUP_DETECTOR_PERF if HAVE_HARDLOCKUP_DETECTOR_PERF - select HARDLOCKUP_DETECTOR_ARCH if HAVE_HARDLOCKUP_DETECTOR_ARCH help Say Y here to enable the kernel to act as a watchdog to detect hard lockups. @@ -978,6 +1156,19 @@ config WQ_WATCHDOG state. This can be configured through kernel parameter "workqueue.watchdog_thresh" and its sysfs counterpart. +config TEST_LOCKUP + tristate "Test module to generate lockups" + depends on m + help + This builds the "test_lockup" module that helps to make sure + that watchdogs and lockup detectors are working properly. + + Depending on module parameters it could emulate soft or hard + lockup, "hung task", or locking arbitrary lock for a long time. + Also it could generate series of lockups with cooling-down periods. + + If unsure, say N. + endmenu # "Debug lockups and hangs" menu "Scheduler Debugging" @@ -1025,7 +1216,7 @@ config DEBUG_TIMEKEEPING config DEBUG_PREEMPT bool "Debug preemptible kernel" - depends on DEBUG_KERNEL && PREEMPT && TRACE_IRQFLAGS_SUPPORT + depends on DEBUG_KERNEL && PREEMPTION && TRACE_IRQFLAGS_SUPPORT default y help If you say Y here then the kernel will use a debug variant of the @@ -1045,11 +1236,12 @@ config PROVE_LOCKING depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT select LOCKDEP select DEBUG_SPINLOCK - select DEBUG_MUTEXES + select DEBUG_MUTEXES if !PREEMPT_RT select DEBUG_RT_MUTEXES if RT_MUTEXES select DEBUG_RWSEMS select DEBUG_WW_MUTEX_SLOWPATH select DEBUG_LOCK_ALLOC + select PREEMPT_COUNT if !ARCH_NO_PREEMPT select TRACE_IRQFLAGS default n help @@ -1086,12 +1278,29 @@ config PROVE_LOCKING For more details, see Documentation/locking/lockdep-design.rst. +config PROVE_RAW_LOCK_NESTING + bool "Enable raw_spinlock - spinlock nesting checks" + depends on PROVE_LOCKING + default n + help + Enable the raw_spinlock vs. spinlock nesting checks which ensure + that the lock nesting rules for PREEMPT_RT enabled kernels are + not violated. + + NOTE: There are known nesting problems. So if you enable this + option expect lockdep splats until these problems have been fully + addressed which is work in progress. This config switch allows to + identify and analyze these problems. It will be removed and the + check permanently enabled once the main issues have been fixed. + + If unsure, select N. + config LOCK_STAT bool "Lock usage statistics" depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT select LOCKDEP select DEBUG_SPINLOCK - select DEBUG_MUTEXES + select DEBUG_MUTEXES if !PREEMPT_RT select DEBUG_RT_MUTEXES if RT_MUTEXES select DEBUG_LOCK_ALLOC default n @@ -1127,7 +1336,7 @@ config DEBUG_SPINLOCK config DEBUG_MUTEXES bool "Mutex debugging: basic checks" - depends on DEBUG_KERNEL + depends on DEBUG_KERNEL && !PREEMPT_RT help This feature allows mutex semantics violations to be detected and reported. @@ -1137,7 +1346,8 @@ config DEBUG_WW_MUTEX_SLOWPATH depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT select DEBUG_LOCK_ALLOC select DEBUG_SPINLOCK - select DEBUG_MUTEXES + select DEBUG_MUTEXES if !PREEMPT_RT + select DEBUG_RT_MUTEXES if PREEMPT_RT help This feature enables slowpath testing for w/w mutex users by injecting additional -EDEADLK wound/backoff cases. Together with @@ -1160,7 +1370,7 @@ config DEBUG_LOCK_ALLOC bool "Lock debugging: detect incorrect freeing of live locks" depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT select DEBUG_SPINLOCK - select DEBUG_MUTEXES + select DEBUG_MUTEXES if !PREEMPT_RT select DEBUG_RT_MUTEXES if RT_MUTEXES select LOCKDEP help @@ -1175,16 +1385,56 @@ config LOCKDEP bool depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT select STACKTRACE - select FRAME_POINTER if !MIPS && !PPC && !ARM && !S390 && !MICROBLAZE && !ARC && !X86 select KALLSYMS select KALLSYMS_ALL config LOCKDEP_SMALL bool +config LOCKDEP_BITS + int "Bitsize for MAX_LOCKDEP_ENTRIES" + depends on LOCKDEP && !LOCKDEP_SMALL + range 10 30 + default 15 + help + Try increasing this value if you hit "BUG: MAX_LOCKDEP_ENTRIES too low!" message. + +config LOCKDEP_CHAINS_BITS + int "Bitsize for MAX_LOCKDEP_CHAINS" + depends on LOCKDEP && !LOCKDEP_SMALL + range 10 30 + default 16 + help + Try increasing this value if you hit "BUG: MAX_LOCKDEP_CHAINS too low!" message. + +config LOCKDEP_STACK_TRACE_BITS + int "Bitsize for MAX_STACK_TRACE_ENTRIES" + depends on LOCKDEP && !LOCKDEP_SMALL + range 10 30 + default 19 + help + Try increasing this value if you hit "BUG: MAX_STACK_TRACE_ENTRIES too low!" message. + +config LOCKDEP_STACK_TRACE_HASH_BITS + int "Bitsize for STACK_TRACE_HASH_SIZE" + depends on LOCKDEP && !LOCKDEP_SMALL + range 10 30 + default 14 + help + Try increasing this value if you need large MAX_STACK_TRACE_ENTRIES. + +config LOCKDEP_CIRCULAR_QUEUE_BITS + int "Bitsize for elements in circular_queue struct" + depends on LOCKDEP + range 10 30 + default 12 + help + Try increasing this value if you hit "lockdep bfs error:-1" warning due to __cq_enqueue() failure. + config DEBUG_LOCKDEP bool "Lock dependency engine debugging" depends on DEBUG_KERNEL && LOCKDEP + select DEBUG_IRQFLAGS help If you say Y here, the lock dependency engine will do additional runtime checks to debug itself, at the price @@ -1208,7 +1458,7 @@ config DEBUG_LOCKING_API_SELFTESTS Say Y here if you want the kernel to run a short self-test during bootup. The self-test checks whether common types of locking bugs are detected by debugging mechanisms or not. (if you disable - lock debugging then those bugs wont be detected of course.) + lock debugging then those bugs won't be detected of course.) The following locking APIs are covered: spinlocks, rwlocks, mutexes and rwsems. @@ -1238,14 +1488,48 @@ config WW_MUTEX_SELFTEST Say M if you want these self tests to build as a module. Say N if you are unsure. +config SCF_TORTURE_TEST + tristate "torture tests for smp_call_function*()" + depends on DEBUG_KERNEL + select TORTURE_TEST + help + This option provides a kernel module that runs torture tests + on the smp_call_function() family of primitives. The kernel + module may be built after the fact on the running kernel to + be tested, if desired. + +config CSD_LOCK_WAIT_DEBUG + bool "Debugging for csd_lock_wait(), called from smp_call_function*()" + depends on DEBUG_KERNEL + depends on 64BIT + default n + help + This option enables debug prints when CPUs are slow to respond + to the smp_call_function*() IPI wrappers. These debug prints + include the IPI handler function currently executing (if any) + and relevant stack traces. + endmenu # lock debugging config TRACE_IRQFLAGS + depends on TRACE_IRQFLAGS_SUPPORT bool help Enables hooks to interrupt enabling and disabling for either tracing or lock debugging. +config TRACE_IRQFLAGS_NMI + def_bool y + depends on TRACE_IRQFLAGS + depends on TRACE_IRQFLAGS_NMI_SUPPORT + +config DEBUG_IRQFLAGS + bool "Debug IRQ flag manipulation" + help + Enables checks for potentially unsafe enabling or disabling of + interrupts, such as calling raw_local_irq_restore() when interrupts + are enabled. + config STACKTRACE bool "Stack backtrace support" depends on STACKTRACE_SUPPORT @@ -1397,33 +1681,6 @@ config DEBUG_WQ_FORCE_RR_CPU feature by default. When enabled, memory and cache locality will be impacted. -config DEBUG_BLOCK_EXT_DEVT - bool "Force extended block device numbers and spread them" - depends on DEBUG_KERNEL - depends on BLOCK - default n - help - BIG FAT WARNING: ENABLING THIS OPTION MIGHT BREAK BOOTING ON - SOME DISTRIBUTIONS. DO NOT ENABLE THIS UNLESS YOU KNOW WHAT - YOU ARE DOING. Distros, please enable this and fix whatever - is broken. - - Conventionally, block device numbers are allocated from - predetermined contiguous area. However, extended block area - may introduce non-contiguous block device numbers. This - option forces most block device numbers to be allocated from - the extended space and spreads them to discover kernel or - userland code paths which assume predetermined contiguous - device number allocation. - - Note that turning on this debug option shuffles all the - device numbers for all IDE and SCSI devices including libata - ones, so root partition specified using device number - directly (via rdev or root=MAJ:MIN) won't work anymore. - Textual device names (root=/dev/sdXn) will continue to work. - - Say N if you are unsure. - config CPU_HOTPLUG_STATE_CONTROL bool "Enable CPU hotplug state control" depends on DEBUG_KERNEL @@ -1442,12 +1699,11 @@ config LATENCYTOP depends on DEBUG_KERNEL depends on STACKTRACE_SUPPORT depends on PROC_FS - select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM && !ARC && !X86 + depends on FRAME_POINTER || MIPS || PPC || S390 || MICROBLAZE || ARM || ARC || X86 select KALLSYMS select KALLSYMS_ALL select STACKTRACE select SCHEDSTATS - select SCHED_DEBUG help Enable this option if you want to use the LatencyTOP tool to find out which userspace is blocking on what kernel operations. @@ -1481,7 +1737,7 @@ config PROVIDE_OHCI1394_DMA_INIT This code (~1k) is freed after boot. By then, the firewire stack in charge of the OHCI-1394 controllers should be used instead. - See Documentation/debugging-via-ohci1394.txt for more information. + See Documentation/core-api/debugging-via-ohci1394.rst for more information. source "samples/Kconfig" @@ -1491,7 +1747,7 @@ config ARCH_HAS_DEVMEM_IS_ALLOWED config STRICT_DEVMEM bool "Filter access to /dev/mem" depends on MMU && DEVMEM - depends on ARCH_HAS_DEVMEM_IS_ALLOWED + depends on ARCH_HAS_DEVMEM_IS_ALLOWED || GENERIC_LIB_DEVMEM_IS_ALLOWED default y if PPC || X86 || ARM64 help If this option is disabled, you allow userspace (root) access to all @@ -1628,11 +1884,18 @@ config FAILSLAB Provide fault-injection capability for kmalloc. config FAIL_PAGE_ALLOC - bool "Fault-injection capabilitiy for alloc_pages()" + bool "Fault-injection capability for alloc_pages()" depends on FAULT_INJECTION help Provide fault-injection capability for alloc_pages(). +config FAULT_INJECTION_USERCOPY + bool "Fault injection capability for usercopy functions" + depends on FAULT_INJECTION + help + Provides fault-injection capability to inject failures + in usercopy functions (copy_from_user(), get_user(), ...). + config FAIL_MAKE_REQUEST bool "Fault-injection capability for disk IO" depends on FAULT_INJECTION && BLOCK @@ -1648,7 +1911,7 @@ config FAIL_IO_TIMEOUT thus exercising the error handling. Only works with drivers that use the generic timeout handling, - for others it wont do anything. + for others it won't do anything. config FAIL_FUTEX bool "Fault-injection capability for futexes" @@ -1683,12 +1946,19 @@ config FAIL_MMC_REQUEST and to test how the mmc host driver handles retries from the block device. +config FAIL_SUNRPC + bool "Fault-injection capability for SunRPC" + depends on FAULT_INJECTION_DEBUG_FS && SUNRPC_DEBUG + help + Provide fault-injection capability for SunRPC and + its consumers. + config FAULT_INJECTION_STACKTRACE_FILTER bool "stacktrace filter for fault-injection capabilities" depends on FAULT_INJECTION_DEBUG_FS && STACKTRACE_SUPPORT depends on !X86_64 select STACKTRACE - select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM && !ARC && !X86 + depends on FRAME_POINTER || MIPS || PPC || S390 || MICROBLAZE || ARM || ARC || X86 help Provide stacktrace filter for fault-injection capabilities @@ -1740,6 +2010,15 @@ config KCOV_INSTRUMENT_ALL filesystem fuzzing with AFL) then you will want to enable coverage for more specific subsets of files, and should say n here. +config KCOV_IRQ_AREA_SIZE + hex "Size of interrupt coverage collection area in words" + depends on KCOV + default 0x40000 + help + KCOV uses preallocated per-cpu areas to collect coverage from + soft interrupts. This specifies the size of those areas in the + number of unsigned long words. + menuconfig RUNTIME_TESTING_MENU bool "Runtime Testing" def_bool y @@ -1760,8 +2039,9 @@ config LKDTM Documentation/fault-injection/provoke-crashes.rst config TEST_LIST_SORT - tristate "Linked list sorting test" - depends on DEBUG_KERNEL || m + tristate "Linked list sorting test" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS help Enable this to turn on 'list_sort()' function test. This test is executed only once during system boot (so affects only boot time), @@ -1769,19 +2049,41 @@ config TEST_LIST_SORT If unsure, say N. -config TEST_SORT - tristate "Array-based sort test" +config TEST_MIN_HEAP + tristate "Min heap test" depends on DEBUG_KERNEL || m help + Enable this to turn on min heap function tests. This test is + executed only once during system boot (so affects only boot time), + or at module load time. + + If unsure, say N. + +config TEST_SORT + tristate "Array-based sort test" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help This option enables the self-test function of 'sort()' at boot, or at module load time. If unsure, say N. +config TEST_DIV64 + tristate "64bit/32bit division and modulo test" + depends on DEBUG_KERNEL || m + help + Enable this to turn on 'do_div()' function test. This test is + executed only once during system boot (so affects only boot time), + or at module load time. + + If unsure, say N. + config KPROBES_SANITY_TEST - bool "Kprobes sanity tests" + tristate "Kprobes sanity tests" depends on DEBUG_KERNEL depends on KPROBES + depends on KUNIT help This option provides for testing basic kprobes functionality on boot. Samples of kprobe and kretprobe are inserted and @@ -1850,7 +2152,7 @@ config ASYNC_RAID6_TEST tristate "Self test for hardware accelerated raid6 recovery" depends on ASYNC_RAID6_RECOV select ASYNC_MEMCPY - ---help--- + help This is a one-shot self test that permutes through the recovery of all the possible two disk failure scenarios for a N-disk array. Recovery is performed with the asynchronous @@ -1862,6 +2164,9 @@ config ASYNC_RAID6_TEST config TEST_HEXDUMP tristate "Test functions located in the hexdump module at runtime" +config STRING_SELFTEST + tristate "Test string functions at runtime" + config TEST_STRING_HELPERS tristate "Test functions located in the string_helpers module at runtime" @@ -1874,6 +2179,9 @@ config TEST_KSTRTOX config TEST_PRINTF tristate "Test printf() family of functions at runtime" +config TEST_SCANF + tristate "Test scanf() family of functions at runtime" + config TEST_BITMAP tristate "Test bitmap_*() family of functions at runtime" help @@ -1881,13 +2189,6 @@ config TEST_BITMAP If unsure, say N. -config TEST_BITFIELD - tristate "Test bitfield functions at runtime" - help - Enable this option to test the bitfield functions at boot. - - If unsure, say N. - config TEST_UUID tristate "Test functions located in the uuid module at runtime" @@ -1947,6 +2248,19 @@ config TEST_LKM If unsure, say N. +config TEST_BITOPS + tristate "Test module for compilation of bitops operations" + depends on m + help + This builds the "test_bitops" module that is much like the + TEST_LKM module except that it does a basic exercise of the + set/clear_bit macros and get_count_order/long to make sure there are + no compiler warnings from C=1 sparse checker or -Wextra + compilations. It has no dependencies and doesn't run or load unless + explicitly requested by name. for example: modprobe test_bitops. + + If unsure, say N. + config TEST_VMALLOC tristate "Test module for stress/performance analysis of vmalloc allocator" default n @@ -2024,9 +2338,37 @@ config TEST_SYSCTL If unsure, say N. +config BITFIELD_KUNIT + tristate "KUnit test bitfield functions at runtime" + depends on KUNIT + help + Enable this option to test the bitfield functions at boot. + + KUnit tests run during boot and output the results to the debug log + in TAP format (http://testanything.org/). Only useful for kernel devs + running the KUnit test harness, and not intended for inclusion into a + production build. + + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + +config RESOURCE_KUNIT_TEST + tristate "KUnit test for resource API" + depends on KUNIT + help + This builds the resource API unit test. + Tests the logic of API provided by resource.c and ioport.h. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + config SYSCTL_KUNIT_TEST - bool "KUnit test for sysctl" + tristate "KUnit test for sysctl" if !KUNIT_ALL_TESTS depends on KUNIT + default KUNIT_ALL_TESTS help This builds the proc sysctl unit test, which runs on boot. Tests the API contract and implementation correctness of sysctl. @@ -2036,15 +2378,16 @@ config SYSCTL_KUNIT_TEST If unsure, say N. config LIST_KUNIT_TEST - bool "KUnit Test for Kernel Linked-list structures" + tristate "KUnit Test for Kernel Linked-list structures" if !KUNIT_ALL_TESTS depends on KUNIT + default KUNIT_ALL_TESTS help This builds the linked list KUnit test suite. It tests that the API and basic functionality of the list_head type and associated macros. KUnit tests run during boot and output the results to the debug log - in TAP format (http://testanything.org/). Only useful for kernel devs + in TAP format (https://testanything.org/). Only useful for kernel devs running the KUnit test harness, and not intended for inclusion into a production build. @@ -2053,6 +2396,74 @@ config LIST_KUNIT_TEST If unsure, say N. +config LINEAR_RANGES_TEST + tristate "KUnit test for linear_ranges" + depends on KUNIT + select LINEAR_RANGES + help + This builds the linear_ranges unit test, which runs on boot. + Tests the linear_ranges logic correctness. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + +config CMDLINE_KUNIT_TEST + tristate "KUnit test for cmdline API" + depends on KUNIT + help + This builds the cmdline API unit test. + Tests the logic of API provided by cmdline.c. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + +config BITS_TEST + tristate "KUnit test for bits.h" + depends on KUNIT + help + This builds the bits unit test. + Tests the logic of macros defined in bits.h. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + +config SLUB_KUNIT_TEST + tristate "KUnit test for SLUB cache error detection" if !KUNIT_ALL_TESTS + depends on SLUB_DEBUG && KUNIT + default KUNIT_ALL_TESTS + help + This builds SLUB allocator unit test. + Tests SLUB cache debugging functionality. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + +config RATIONAL_KUNIT_TEST + tristate "KUnit test for rational.c" if !KUNIT_ALL_TESTS + depends on KUNIT && RATIONAL + default KUNIT_ALL_TESTS + help + This builds the rational math unit test. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + +config MEMCPY_KUNIT_TEST + tristate "Test memcpy(), memmove(), and memset() functions at runtime" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Builds unit tests for memcpy(), memmove(), and memset() functions. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + config TEST_UDELAY tristate "udelay test driver" help @@ -2162,13 +2573,65 @@ config TEST_MEMINIT If unsure, say N. +config TEST_HMM + tristate "Test HMM (Heterogeneous Memory Management)" + depends on TRANSPARENT_HUGEPAGE + depends on DEVICE_PRIVATE + select HMM_MIRROR + select MMU_NOTIFIER + help + This is a pseudo device driver solely for testing HMM. + Say M here if you want to build the HMM test module. + Doing so will allow you to run tools/testing/selftest/vm/hmm-tests. + + If unsure, say N. + +config TEST_FREE_PAGES + tristate "Test freeing pages" + help + Test that a memory leak does not occur due to a race between + freeing a block of pages and a speculative page reference. + Loading this module is safe if your kernel has the bug fixed. + If the bug is not fixed, it will leak gigabytes of memory and + probably OOM your system. + +config TEST_FPU + tristate "Test floating point operations in kernel space" + depends on X86 && !KCOV_INSTRUMENT_ALL + help + Enable this option to add /sys/kernel/debug/selftest_helpers/test_fpu + which will trigger a sequence of floating point operations. This is used + for self-testing floating point control register setting in + kernel_fpu_begin(). + + If unsure, say N. + +config TEST_CLOCKSOURCE_WATCHDOG + tristate "Test clocksource watchdog in kernel space" + depends on CLOCKSOURCE_WATCHDOG + help + Enable this option to create a kernel module that will trigger + a test of the clocksource watchdog. This module may be loaded + via modprobe or insmod in which case it will run upon being + loaded, or it may be built in, in which case it will run + shortly after boot. + + If unsure, say N. + endif # RUNTIME_TESTING_MENU +config ARCH_USE_MEMTEST + bool + help + An architecture should select this when it uses early_memtest() + during boot process. + config MEMTEST bool "Memtest" - ---help--- + depends on ARCH_USE_MEMTEST + help This option adds a kernel parameter 'memtest', which allows memtest - to be set. + to be set and executed. memtest=0, mean disabled; -- default memtest=1, mean do 1 test pattern; ... @@ -2186,4 +2649,6 @@ config HYPERV_TESTING endmenu # "Kernel Testing and Coverage" +source "Documentation/Kconfig" + endmenu # Kernel hacking diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index 81f5464ea9e1..cdc842d090db 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -6,85 +6,126 @@ config HAVE_ARCH_KASAN config HAVE_ARCH_KASAN_SW_TAGS bool -config HAVE_ARCH_KASAN_VMALLOC +config HAVE_ARCH_KASAN_HW_TAGS bool +config HAVE_ARCH_KASAN_VMALLOC + bool + +config ARCH_DISABLE_KASAN_INLINE + bool + help + An architecture might not support inline instrumentation. + When this option is selected, inline and stack instrumentation are + disabled. + config CC_HAS_KASAN_GENERIC def_bool $(cc-option, -fsanitize=kernel-address) config CC_HAS_KASAN_SW_TAGS def_bool $(cc-option, -fsanitize=kernel-hwaddress) -config KASAN +# This option is only required for software KASAN modes. +# Old GCC versions don't have proper support for no_sanitize_address. +# See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89124 for details. +config CC_HAS_WORKING_NOSANITIZE_ADDRESS + def_bool !CC_IS_GCC || GCC_VERSION >= 80300 + +menuconfig KASAN bool "KASAN: runtime memory debugger" - depends on (HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC) || \ - (HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS) + depends on (((HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC) || \ + (HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS)) && \ + CC_HAS_WORKING_NOSANITIZE_ADDRESS) || \ + HAVE_ARCH_KASAN_HW_TAGS depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB) + select STACKDEPOT help Enables KASAN (KernelAddressSANitizer) - runtime memory debugger, designed to find out-of-bounds accesses and use-after-free bugs. See Documentation/dev-tools/kasan.rst for details. +if KASAN + choice prompt "KASAN mode" - depends on KASAN default KASAN_GENERIC help - KASAN has two modes: generic KASAN (similar to userspace ASan, - x86_64/arm64/xtensa, enabled with CONFIG_KASAN_GENERIC) and - software tag-based KASAN (a version based on software memory - tagging, arm64 only, similar to userspace HWASan, enabled with - CONFIG_KASAN_SW_TAGS). - Both generic and tag-based KASAN are strictly debugging features. + KASAN has three modes: + 1. generic KASAN (similar to userspace ASan, + x86_64/arm64/xtensa, enabled with CONFIG_KASAN_GENERIC), + 2. software tag-based KASAN (arm64 only, based on software + memory tagging (similar to userspace HWASan), enabled with + CONFIG_KASAN_SW_TAGS), and + 3. hardware tag-based KASAN (arm64 only, based on hardware + memory tagging, enabled with CONFIG_KASAN_HW_TAGS). + + All KASAN modes are strictly debugging features. + + For better error reports enable CONFIG_STACKTRACE. config KASAN_GENERIC bool "Generic mode" depends on HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC - depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB) + depends on CC_HAS_WORKING_NOSANITIZE_ADDRESS select SLUB_DEBUG if SLUB select CONSTRUCTORS - select STACKDEPOT help Enables generic KASAN mode. - Supported in both GCC and Clang. With GCC it requires version 4.9.2 - or later for basic support and version 5.0 or later for detection of - out-of-bounds accesses for stack and global variables and for inline - instrumentation mode (CONFIG_KASAN_INLINE). With Clang it requires - version 3.7.0 or later and it doesn't support detection of - out-of-bounds accesses for global variables yet. + + This mode is supported in both GCC and Clang. With GCC it requires + version 8.3.0 or later. Any supported Clang version is compatible, + but detection of out-of-bounds accesses for global variables is + supported only since Clang 11. + This mode consumes about 1/8th of available memory at kernel start and introduces an overhead of ~x1.5 for the rest of the allocations. The performance slowdown is ~x3. - For better error detection enable CONFIG_STACKTRACE. + Currently CONFIG_KASAN_GENERIC doesn't work with CONFIG_DEBUG_SLAB (the resulting kernel does not boot). config KASAN_SW_TAGS bool "Software tag-based mode" depends on HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS - depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB) + depends on CC_HAS_WORKING_NOSANITIZE_ADDRESS select SLUB_DEBUG if SLUB select CONSTRUCTORS - select STACKDEPOT help Enables software tag-based KASAN mode. - This mode requires Top Byte Ignore support by the CPU and therefore - is only supported for arm64. - This mode requires Clang version 7.0.0 or later. + + This mode require software memory tagging support in the form of + HWASan-like compiler instrumentation. + + Currently this mode is only implemented for arm64 CPUs and relies on + Top Byte Ignore. This mode requires Clang. + This mode consumes about 1/16th of available memory at kernel start and introduces an overhead of ~20% for the rest of the allocations. This mode may potentially introduce problems relating to pointer casting and comparison, as it embeds tags into the top byte of each pointer. - For better error detection enable CONFIG_STACKTRACE. + Currently CONFIG_KASAN_SW_TAGS doesn't work with CONFIG_DEBUG_SLAB (the resulting kernel does not boot). +config KASAN_HW_TAGS + bool "Hardware tag-based mode" + depends on HAVE_ARCH_KASAN_HW_TAGS + depends on SLUB + help + Enables hardware tag-based KASAN mode. + + This mode requires hardware memory tagging support, and can be used + by any architecture that provides it. + + Currently this mode is only implemented for arm64 CPUs starting from + ARMv8.5 and relies on Memory Tagging Extension and Top Byte Ignore. + endchoice choice prompt "Instrumentation type" - depends on KASAN + depends on KASAN_GENERIC || KASAN_SW_TAGS default KASAN_OUTLINE config KASAN_OUTLINE @@ -98,18 +139,20 @@ config KASAN_OUTLINE config KASAN_INLINE bool "Inline instrumentation" + depends on !ARCH_DISABLE_KASAN_INLINE help Compiler directly inserts code checking shadow memory before memory accesses. This is faster than outline (in some workloads it gives about x2 boost over outline instrumentation), but make kernel's .text size much bigger. - For CONFIG_KASAN_GENERIC this requires GCC 5.0 or later. endchoice -config KASAN_STACK_ENABLE +config KASAN_STACK bool "Enable stack instrumentation (unsafe)" if CC_IS_CLANG && !COMPILE_TEST - depends on KASAN + depends on KASAN_GENERIC || KASAN_SW_TAGS + depends on !ARCH_DISABLE_KASAN_INLINE + default y if CC_IS_GCC help The LLVM stack address sanitizer has a know problem that causes excessive stack usage in a lot of functions, see @@ -122,24 +165,13 @@ config KASAN_STACK_ENABLE but clang users can still enable it for builds without CONFIG_COMPILE_TEST. On gcc it is assumed to always be safe to use and enabled by default. + If the architecture disables inline instrumentation, stack + instrumentation is also disabled as it adds inline-style + instrumentation that is run unconditionally. -config KASAN_STACK - int - default 1 if KASAN_STACK_ENABLE || CC_IS_GCC - default 0 - -config KASAN_S390_4_LEVEL_PAGING - bool "KASan: use 4-level paging" - depends on KASAN && S390 - help - Compiling the kernel with KASan disables automatic 3-level vs - 4-level paging selection. 3-level paging is used by default (up - to 3TB of RAM with KASan enabled). This options allows to force - 4-level paging instead. - -config KASAN_SW_TAGS_IDENTIFY +config KASAN_TAGS_IDENTIFY bool "Enable memory corruption identification" - depends on KASAN_SW_TAGS + depends on KASAN_SW_TAGS || KASAN_HW_TAGS help This option enables best-effort identification of bug type (use-after-free or out-of-bounds) at the cost of increased @@ -147,7 +179,7 @@ config KASAN_SW_TAGS_IDENTIFY config KASAN_VMALLOC bool "Back mappings in vmalloc space with real shadow memory" - depends on KASAN && HAVE_ARCH_KASAN_VMALLOC + depends on KASAN_GENERIC && HAVE_ARCH_KASAN_VMALLOC help By default, the shadow region for vmalloc space is the read-only zero page. This means that KASAN cannot detect errors involving @@ -158,10 +190,24 @@ config KASAN_VMALLOC for KASAN to detect more sorts of errors (and to support vmapped stacks), but at the cost of higher memory usage. -config TEST_KASAN - tristate "Module for testing KASAN for bug detection" - depends on m && KASAN +config KASAN_KUNIT_TEST + tristate "KUnit-compatible tests of KASAN bug detection capabilities" if !KUNIT_ALL_TESTS + depends on KASAN && KUNIT + default KUNIT_ALL_TESTS help - This is a test module doing various nasty things like - out of bounds accesses, use after free. It is useful for testing + This is a KUnit test suite doing various nasty things like + out of bounds and use after free accesses. It is useful for testing kernel debugging features like KASAN. + + For more information on KUnit and unit tests in general, please refer + to the KUnit documentation in Documentation/dev-tools/kunit. + +config KASAN_MODULE_TEST + tristate "KUnit-incompatible tests of KASAN bug detection capabilities" + depends on m && KASAN && !KASAN_HW_TAGS + help + This is a part of the KASAN test suite that is incompatible with + KUnit. Currently includes tests that do bad copy_from/to_user + accesses. + +endif # KASAN diff --git a/lib/Kconfig.kcsan b/lib/Kconfig.kcsan new file mode 100644 index 000000000000..e0a93ffdef30 --- /dev/null +++ b/lib/Kconfig.kcsan @@ -0,0 +1,245 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config HAVE_ARCH_KCSAN + bool + +config HAVE_KCSAN_COMPILER + def_bool (CC_IS_CLANG && $(cc-option,-fsanitize=thread -mllvm -tsan-distinguish-volatile=1)) || \ + (CC_IS_GCC && $(cc-option,-fsanitize=thread --param tsan-distinguish-volatile=1)) + help + For the list of compilers that support KCSAN, please see + <file:Documentation/dev-tools/kcsan.rst>. + +config KCSAN_KCOV_BROKEN + def_bool KCOV && CC_HAS_SANCOV_TRACE_PC + depends on CC_IS_CLANG + depends on !$(cc-option,-Werror=unused-command-line-argument -fsanitize=thread -fsanitize-coverage=trace-pc) + help + Some versions of clang support either KCSAN and KCOV but not the + combination of the two. + See https://bugs.llvm.org/show_bug.cgi?id=45831 for the status + in newer releases. + +menuconfig KCSAN + bool "KCSAN: dynamic data race detector" + depends on HAVE_ARCH_KCSAN && HAVE_KCSAN_COMPILER + depends on DEBUG_KERNEL && !KASAN + depends on !KCSAN_KCOV_BROKEN + select STACKTRACE + help + The Kernel Concurrency Sanitizer (KCSAN) is a dynamic + data-race detector that relies on compile-time instrumentation. + KCSAN uses a watchpoint-based sampling approach to detect races. + + While KCSAN's primary purpose is to detect data races, it + also provides assertions to check data access constraints. + These assertions can expose bugs that do not manifest as + data races. + + See <file:Documentation/dev-tools/kcsan.rst> for more details. + +if KCSAN + +config CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE + def_bool (CC_IS_CLANG && $(cc-option,-fsanitize=thread -mllvm -tsan-compound-read-before-write=1)) || \ + (CC_IS_GCC && $(cc-option,-fsanitize=thread --param tsan-compound-read-before-write=1)) + help + The compiler instruments plain compound read-write operations + differently (++, --, +=, -=, |=, &=, etc.), which allows KCSAN to + distinguish them from other plain accesses. This is currently + supported by Clang 12 or later. + +config KCSAN_VERBOSE + bool "Show verbose reports with more information about system state" + depends on PROVE_LOCKING + help + If enabled, reports show more information about the system state that + may help better analyze and debug races. This includes held locks and + IRQ trace events. + + While this option should generally be benign, we call into more + external functions on report generation; if a race report is + generated from any one of them, system stability may suffer due to + deadlocks or recursion. If in doubt, say N. + +config KCSAN_SELFTEST + bool "Perform short selftests on boot" + default y + help + Run KCSAN selftests on boot. On test failure, causes the kernel to + panic. Recommended to be enabled, ensuring critical functionality + works as intended. + +config KCSAN_KUNIT_TEST + tristate "KCSAN test for integrated runtime behaviour" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + depends on TRACEPOINTS && KUNIT + select TORTURE_TEST + help + KCSAN test focusing on behaviour of the integrated runtime. Tests + various race scenarios, and verifies the reports generated to + console. Makes use of KUnit for test organization, and the Torture + framework for test thread control. + + Each test case may run at least up to KCSAN_REPORT_ONCE_IN_MS + milliseconds. Test run duration may be optimized by building the + kernel and KCSAN test with KCSAN_REPORT_ONCE_IN_MS set to a lower + than default value. + + Say Y here if you want the test to be built into the kernel and run + during boot; say M if you want the test to build as a module; say N + if you are unsure. + +config KCSAN_EARLY_ENABLE + bool "Early enable during boot" + default y + help + If KCSAN should be enabled globally as soon as possible. KCSAN can + later be enabled/disabled via debugfs. + +config KCSAN_NUM_WATCHPOINTS + int "Number of available watchpoints" + default 64 + help + Total number of available watchpoints. An address range maps into a + specific watchpoint slot as specified in kernel/kcsan/encoding.h. + Although larger number of watchpoints may not be usable due to + limited number of CPUs, a larger value helps to improve performance + due to reducing cache-line contention. The chosen default is a + conservative value; we should almost never observe "no_capacity" + events (see /sys/kernel/debug/kcsan). + +config KCSAN_UDELAY_TASK + int "Delay in microseconds (for tasks)" + default 80 + help + For tasks, the microsecond delay after setting up a watchpoint. + +config KCSAN_UDELAY_INTERRUPT + int "Delay in microseconds (for interrupts)" + default 20 + help + For interrupts, the microsecond delay after setting up a watchpoint. + Interrupts have tighter latency requirements, and their delay should + be lower than for tasks. + +config KCSAN_DELAY_RANDOMIZE + bool "Randomize above delays" + default y + help + If delays should be randomized, where the maximum is KCSAN_UDELAY_*. + If false, the chosen delays are always the KCSAN_UDELAY_* values + as defined above. + +config KCSAN_SKIP_WATCH + int "Skip instructions before setting up watchpoint" + default 4000 + help + The number of per-CPU memory operations to skip, before another + watchpoint is set up, i.e. one in KCSAN_WATCH_SKIP per-CPU + memory operations are used to set up a watchpoint. A smaller value + results in more aggressive race detection, whereas a larger value + improves system performance at the cost of missing some races. + +config KCSAN_SKIP_WATCH_RANDOMIZE + bool "Randomize watchpoint instruction skip count" + default y + help + If instruction skip count should be randomized, where the maximum is + KCSAN_WATCH_SKIP. If false, the chosen value is always + KCSAN_WATCH_SKIP. + +config KCSAN_INTERRUPT_WATCHER + bool "Interruptible watchers" if !KCSAN_STRICT + default KCSAN_STRICT + help + If enabled, a task that set up a watchpoint may be interrupted while + delayed. This option will allow KCSAN to detect races between + interrupted tasks and other threads of execution on the same CPU. + + Currently disabled by default, because not all safe per-CPU access + primitives and patterns may be accounted for, and therefore could + result in false positives. + +config KCSAN_REPORT_ONCE_IN_MS + int "Duration in milliseconds, in which any given race is only reported once" + default 3000 + help + Any given race is only reported once in the defined time window. + Different races may still generate reports within a duration that is + smaller than the duration defined here. This allows rate limiting + reporting to avoid flooding the console with reports. Setting this + to 0 disables rate limiting. + +# The main purpose of the below options is to control reported data races, and +# are not expected to be switched frequently by non-testers or at runtime. +# The defaults are chosen to be conservative, and can miss certain bugs. + +config KCSAN_REPORT_RACE_UNKNOWN_ORIGIN + bool "Report races of unknown origin" + default y + help + If KCSAN should report races where only one access is known, and the + conflicting access is of unknown origin. This type of race is + reported if it was only possible to infer a race due to a data value + change while an access is being delayed on a watchpoint. + +config KCSAN_STRICT + bool "Strict data-race checking" + help + KCSAN will report data races with the strictest possible rules, which + closely aligns with the rules defined by the Linux-kernel memory + consistency model (LKMM). + +config KCSAN_REPORT_VALUE_CHANGE_ONLY + bool "Only report races where watcher observed a data value change" + default y + depends on !KCSAN_STRICT + help + If enabled and a conflicting write is observed via a watchpoint, but + the data value of the memory location was observed to remain + unchanged, do not report the data race. + +config KCSAN_ASSUME_PLAIN_WRITES_ATOMIC + bool "Assume that plain aligned writes up to word size are atomic" + default y + depends on !KCSAN_STRICT + help + Assume that plain aligned writes up to word size are atomic by + default, and also not subject to other unsafe compiler optimizations + resulting in data races. This will cause KCSAN to not report data + races due to conflicts where the only plain accesses are aligned + writes up to word size: conflicts between marked reads and plain + aligned writes up to word size will not be reported as data races; + notice that data races between two conflicting plain aligned writes + will also not be reported. + +config KCSAN_IGNORE_ATOMICS + bool "Do not instrument marked atomic accesses" + depends on !KCSAN_STRICT + help + Never instrument marked atomic accesses. This option can be used for + additional filtering. Conflicting marked atomic reads and plain + writes will never be reported as a data race, however, will cause + plain reads and marked writes to result in "unknown origin" reports. + If combined with CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN=n, data + races where at least one access is marked atomic will never be + reported. + + Similar to KCSAN_ASSUME_PLAIN_WRITES_ATOMIC, but including unaligned + accesses, conflicting marked atomic reads and plain writes will not + be reported as data races; however, unlike that option, data races + due to two conflicting plain writes will be reported (aligned and + unaligned, if CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=n). + +config KCSAN_PERMISSIVE + bool "Enable all additional permissive rules" + depends on KCSAN_REPORT_VALUE_CHANGE_ONLY + help + Enable additional permissive rules to ignore certain classes of data + races (also see kernel/kcsan/permissive.h). None of the permissive + rules imply that such data races are generally safe, but can be used + to further reduce reported data races due to data-racy patterns + common across the kernel. + +endif # KCSAN diff --git a/lib/Kconfig.kfence b/lib/Kconfig.kfence new file mode 100644 index 000000000000..912f252a41fc --- /dev/null +++ b/lib/Kconfig.kfence @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config HAVE_ARCH_KFENCE + bool + +menuconfig KFENCE + bool "KFENCE: low-overhead sampling-based memory safety error detector" + depends on HAVE_ARCH_KFENCE && (SLAB || SLUB) + select STACKTRACE + select IRQ_WORK + help + KFENCE is a low-overhead sampling-based detector of heap out-of-bounds + access, use-after-free, and invalid-free errors. KFENCE is designed + to have negligible cost to permit enabling it in production + environments. + + See <file:Documentation/dev-tools/kfence.rst> for more details. + + Note that, KFENCE is not a substitute for explicit testing with tools + such as KASAN. KFENCE can detect a subset of bugs that KASAN can + detect, albeit at very different performance profiles. If you can + afford to use KASAN, continue using KASAN, for example in test + environments. If your kernel targets production use, and cannot + enable KASAN due to its cost, consider using KFENCE. + +if KFENCE + +config KFENCE_SAMPLE_INTERVAL + int "Default sample interval in milliseconds" + default 100 + help + The KFENCE sample interval determines the frequency with which heap + allocations will be guarded by KFENCE. May be overridden via boot + parameter "kfence.sample_interval". + + Set this to 0 to disable KFENCE by default, in which case only + setting "kfence.sample_interval" to a non-zero value enables KFENCE. + +config KFENCE_NUM_OBJECTS + int "Number of guarded objects available" + range 1 65535 + default 255 + help + The number of guarded objects available. For each KFENCE object, 2 + pages are required; with one containing the object and two adjacent + ones used as guard pages. + +config KFENCE_STATIC_KEYS + bool "Use static keys to set up allocations" if EXPERT + depends on JUMP_LABEL + help + Use static keys (static branches) to set up KFENCE allocations. This + option is only recommended when using very large sample intervals, or + performance has carefully been evaluated with this option. + + Using static keys comes with trade-offs that need to be carefully + evaluated given target workloads and system architectures. Notably, + enabling and disabling static keys invoke IPI broadcasts, the latency + and impact of which is much harder to predict than a dynamic branch. + + Say N if you are unsure. + +config KFENCE_STRESS_TEST_FAULTS + int "Stress testing of fault handling and error reporting" if EXPERT + default 0 + help + The inverse probability with which to randomly protect KFENCE object + pages, resulting in spurious use-after-frees. The main purpose of + this option is to stress test KFENCE with concurrent error reports + and allocations/frees. A value of 0 disables stress testing logic. + + Only for KFENCE testing; set to 0 if you are not a KFENCE developer. + +config KFENCE_KUNIT_TEST + tristate "KFENCE integration test suite" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + depends on TRACEPOINTS && KUNIT + help + Test suite for KFENCE, testing various error detection scenarios with + various allocation types, and checking that reports are correctly + output to console. + + Say Y here if you want the test to be built into the kernel and run + during boot; say M if you want the test to build as a module; say N + if you are unsure. + +endif # KFENCE diff --git a/lib/Kconfig.kgdb b/lib/Kconfig.kgdb index 933680b59e2d..05dae05b6cc9 100644 --- a/lib/Kconfig.kgdb +++ b/lib/Kconfig.kgdb @@ -3,6 +3,11 @@ config HAVE_ARCH_KGDB bool +# set if architecture has the its kgdb_arch_handle_qxfer_pkt +# function to enable gdb stub to address XML packet sent from GDB. +config HAVE_ARCH_KGDB_QXFER_PKT + bool + menuconfig KGDB bool "KGDB: kernel debugger" depends on HAVE_ARCH_KGDB @@ -19,6 +24,21 @@ menuconfig KGDB if KGDB +config KGDB_HONOUR_BLOCKLIST + bool "KGDB: use kprobe blocklist to prohibit unsafe breakpoints" + depends on HAVE_KPROBES + depends on MODULES + select KPROBES + default y + help + If set to Y the debug core will use the kprobe blocklist to + identify symbols where it is unsafe to set breakpoints. + In particular this disallows instrumentation of functions + called during debug trap handling and thus makes it very + difficult to inadvertently provoke recursive trap handling. + + If unsure, say Y. + config KGDB_SERIAL_CONSOLE tristate "KGDB: use kgdb over the serial console" select CONSOLE_POLL @@ -124,4 +144,22 @@ config KDB_CONTINUE_CATASTROPHIC CONFIG_KDB_CONTINUE_CATASTROPHIC == 2. KDB forces a reboot. If you are not sure, say 0. +config ARCH_HAS_EARLY_DEBUG + bool + default n + help + If an architecture can definitely handle entering the debugger + when early_param's are parsed then it select this config. + Otherwise, if "kgdbwait" is passed on the kernel command line it + won't actually be processed until dbg_late_init() just after the + call to kgdb_arch_late() is made. + + NOTE: Even if this isn't selected by an architecture we will + still try to register kgdb to handle breakpoints and crashes + when early_param's are parsed, we just won't act on the + "kgdbwait" parameter until dbg_late_init(). If you get a + crash and try to drop into kgdb somewhere between these two + places you might or might not end up being able to use kgdb + depending on exactly how far along the architecture has initted. + endif # KGDB diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index 0e04fcb3ab3d..e5372a13511d 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -2,24 +2,158 @@ config ARCH_HAS_UBSAN_SANITIZE_ALL bool -config UBSAN +menuconfig UBSAN bool "Undefined behaviour sanity checker" help - This option enables undefined behaviour sanity checker + This option enables the Undefined Behaviour sanity checker. Compile-time instrumentation is used to detect various undefined - behaviours in runtime. Various types of checks may be enabled - via boot parameter ubsan_handle - (see: Documentation/dev-tools/ubsan.rst). + behaviours at runtime. For more details, see: + Documentation/dev-tools/ubsan.rst + +if UBSAN + +config UBSAN_TRAP + bool "On Sanitizer warnings, abort the running kernel code" + depends on !COMPILE_TEST + depends on $(cc-option, -fsanitize-undefined-trap-on-error) + help + Building kernels with Sanitizer features enabled tends to grow + the kernel size by around 5%, due to adding all the debugging + text on failure paths. To avoid this, Sanitizer instrumentation + can just issue a trap. This reduces the kernel size overhead but + turns all warnings (including potentially harmless conditions) + into full exceptions that abort the running kernel code + (regardless of context, locks held, etc), which may destabilize + the system. For some system builders this is an acceptable + trade-off. + +config UBSAN_KCOV_BROKEN + def_bool KCOV && CC_HAS_SANCOV_TRACE_PC + depends on CC_IS_CLANG + depends on !$(cc-option,-Werror=unused-command-line-argument -fsanitize=bounds -fsanitize-coverage=trace-pc) + help + Some versions of clang support either UBSAN or KCOV but not the + combination of the two. + See https://bugs.llvm.org/show_bug.cgi?id=45831 for the status + in newer releases. + +config CC_HAS_UBSAN_BOUNDS + def_bool $(cc-option,-fsanitize=bounds) + +config CC_HAS_UBSAN_ARRAY_BOUNDS + def_bool $(cc-option,-fsanitize=array-bounds) + +config UBSAN_BOUNDS + bool "Perform array index bounds checking" + default UBSAN + depends on !UBSAN_KCOV_BROKEN + depends on CC_HAS_UBSAN_ARRAY_BOUNDS || CC_HAS_UBSAN_BOUNDS + help + This option enables detection of directly indexed out of bounds + array accesses, where the array size is known at compile time. + Note that this does not protect array overflows via bad calls + to the {str,mem}*cpy() family of functions (that is addressed + by CONFIG_FORTIFY_SOURCE). + +config UBSAN_ONLY_BOUNDS + def_bool CC_HAS_UBSAN_BOUNDS && !CC_HAS_UBSAN_ARRAY_BOUNDS + depends on UBSAN_BOUNDS + help + This is a weird case: Clang's -fsanitize=bounds includes + -fsanitize=local-bounds, but it's trapping-only, so for + Clang, we must use -fsanitize=array-bounds when we want + traditional array bounds checking enabled. For GCC, we + want -fsanitize=bounds. + +config UBSAN_ARRAY_BOUNDS + def_bool CC_HAS_UBSAN_ARRAY_BOUNDS + depends on UBSAN_BOUNDS + +config UBSAN_LOCAL_BOUNDS + bool "Perform array local bounds checking" + depends on UBSAN_TRAP + depends on !UBSAN_KCOV_BROKEN + depends on $(cc-option,-fsanitize=local-bounds) + help + This option enables -fsanitize=local-bounds which traps when an + exception/error is detected. Therefore, it may only be enabled + with CONFIG_UBSAN_TRAP. + + Enabling this option detects errors due to accesses through a + pointer that is derived from an object of a statically-known size, + where an added offset (which may not be known statically) is + out-of-bounds. + +config UBSAN_SHIFT + bool "Perform checking for bit-shift overflows" + default UBSAN + depends on $(cc-option,-fsanitize=shift) + help + This option enables -fsanitize=shift which checks for bit-shift + operations that overflow to the left or go switch to negative + for signed types. + +config UBSAN_DIV_ZERO + bool "Perform checking for integer divide-by-zero" + depends on $(cc-option,-fsanitize=integer-divide-by-zero) + help + This option enables -fsanitize=integer-divide-by-zero which checks + for integer division by zero. This is effectively redundant with the + kernel's existing exception handling, though it can provide greater + debugging information under CONFIG_UBSAN_REPORT_FULL. + +config UBSAN_UNREACHABLE + bool "Perform checking for unreachable code" + # objtool already handles unreachable checking and gets angry about + # seeing UBSan instrumentation located in unreachable places. + depends on !STACK_VALIDATION + depends on $(cc-option,-fsanitize=unreachable) + help + This option enables -fsanitize=unreachable which checks for control + flow reaching an expected-to-be-unreachable position. + +config UBSAN_OBJECT_SIZE + bool "Perform checking for accesses beyond the end of objects" + default UBSAN + # gcc hugely expands stack usage with -fsanitize=object-size + # https://lore.kernel.org/lkml/CAHk-=wjPasyJrDuwDnpHJS2TuQfExwe=px-SzLeN8GFMAQJPmQ@mail.gmail.com/ + depends on !CC_IS_GCC + depends on $(cc-option,-fsanitize=object-size) + help + This option enables -fsanitize=object-size which checks for accesses + beyond the end of objects where the optimizer can determine both the + object being operated on and its size, usually seen with bad downcasts, + or access to struct members from NULL pointers. + +config UBSAN_BOOL + bool "Perform checking for non-boolean values used as boolean" + default UBSAN + depends on $(cc-option,-fsanitize=bool) + help + This option enables -fsanitize=bool which checks for boolean values being + loaded that are neither 0 nor 1. + +config UBSAN_ENUM + bool "Perform checking for out of bounds enum values" + default UBSAN + depends on $(cc-option,-fsanitize=enum) + help + This option enables -fsanitize=enum which checks for values being loaded + into an enum that are outside the range of given values for the given enum. + +config UBSAN_ALIGNMENT + bool "Perform checking for misaligned pointer usage" + default !HAVE_EFFICIENT_UNALIGNED_ACCESS + depends on !UBSAN_TRAP && !COMPILE_TEST + depends on $(cc-option,-fsanitize=alignment) + help + This option enables the check of unaligned memory accesses. + Enabling this option on architectures that support unaligned + accesses may produce a lot of false positives. config UBSAN_SANITIZE_ALL bool "Enable instrumentation for the entire kernel" - depends on UBSAN depends on ARCH_HAS_UBSAN_SANITIZE_ALL - - # We build with -Wno-maybe-uninitilzed, but we still want to - # use -Wmaybe-uninitilized in allmodconfig builds. - # So dependsy bellow used to disable this option in allmodconfig - depends on !COMPILE_TEST default y help This option activates instrumentation for the entire kernel. @@ -28,22 +162,11 @@ config UBSAN_SANITIZE_ALL Enabling this option will get kernel image size increased significantly. -config UBSAN_NO_ALIGNMENT - bool "Disable checking of pointers alignment" - depends on UBSAN - default y if HAVE_EFFICIENT_UNALIGNED_ACCESS - help - This option disables the check of unaligned memory accesses. - This option should be used when building allmodconfig. - Disabling this option on architectures that support unaligned - accesses may produce a lot of false positives. - -config UBSAN_ALIGNMENT - def_bool !UBSAN_NO_ALIGNMENT - config TEST_UBSAN tristate "Module for testing for undefined behavior detection" - depends on m && UBSAN + depends on m help This is a test module for UBSAN. It triggers various undefined behavior, and detect it. + +endif # if UBSAN diff --git a/lib/Makefile b/lib/Makefile index 93217d44237f..364c23f15578 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -3,10 +3,7 @@ # Makefile for some libs needed in the kernel. # -ifdef CONFIG_FUNCTION_TRACER -ORIG_CFLAGS := $(KBUILD_CFLAGS) -KBUILD_CFLAGS = $(subst $(CC_FLAGS_FTRACE),,$(ORIG_CFLAGS)) -endif +ccflags-remove-$(CONFIG_FUNCTION_TRACER) += $(CC_FLAGS_FTRACE) # These files are disabled because they produce lots of non-interesting and/or # flaky coverage that is not a function of syscall inputs. For example, @@ -16,12 +13,18 @@ KCOV_INSTRUMENT_rbtree.o := n KCOV_INSTRUMENT_list_debug.o := n KCOV_INSTRUMENT_debugobjects.o := n KCOV_INSTRUMENT_dynamic_debug.o := n +KCOV_INSTRUMENT_fault-inject.o := n + +# string.o implements standard library functions like memset/memcpy etc. +# Use -ffreestanding to ensure that the compiler does not try to "optimize" +# them into calls to themselves. +CFLAGS_string.o := -ffreestanding # Early boot use of cmdline, don't instrument it ifdef CONFIG_AMD_MEM_ENCRYPT KASAN_SANITIZE_string.o := n -CFLAGS_string.o := $(call cc-option, -fno-stack-protector) +CFLAGS_string.o += -fno-stack-protector endif lib-y := ctype.o string.o vsprintf.o cmdline.o \ @@ -30,10 +33,10 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ flex_proportions.o ratelimit.o show_mem.o \ is_single_threaded.o plist.o decompress.o kobject_uevent.o \ earlycpio.o seq_buf.o siphash.o dec_and_lock.o \ - nmi_backtrace.o nodemask.o win_minmax.o memcat_p.o + nmi_backtrace.o nodemask.o win_minmax.o memcat_p.o \ + buildid.o lib-$(CONFIG_PRINTK) += dump_stack.o -lib-$(CONFIG_MMU) += ioremap.o lib-$(CONFIG_SMP) += cpumask.o lib-y += kobject.o klist.o @@ -55,17 +58,22 @@ obj-y += kstrtox.o obj-$(CONFIG_FIND_BIT_BENCHMARK) += find_bit_benchmark.o obj-$(CONFIG_TEST_BPF) += test_bpf.o obj-$(CONFIG_TEST_FIRMWARE) += test_firmware.o +obj-$(CONFIG_TEST_BITOPS) += test_bitops.o +CFLAGS_test_bitops.o += -Werror obj-$(CONFIG_TEST_SYSCTL) += test_sysctl.o obj-$(CONFIG_TEST_HASH) += test_hash.o test_siphash.o obj-$(CONFIG_TEST_IDA) += test_ida.o -obj-$(CONFIG_TEST_KASAN) += test_kasan.o +obj-$(CONFIG_KASAN_KUNIT_TEST) += test_kasan.o CFLAGS_test_kasan.o += -fno-builtin CFLAGS_test_kasan.o += $(call cc-disable-warning, vla) +obj-$(CONFIG_KASAN_MODULE_TEST) += test_kasan_module.o +CFLAGS_test_kasan_module.o += -fno-builtin obj-$(CONFIG_TEST_UBSAN) += test_ubsan.o CFLAGS_test_ubsan.o += $(call cc-disable-warning, vla) UBSAN_SANITIZE_test_ubsan.o := y obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o obj-$(CONFIG_TEST_LIST_SORT) += test_list_sort.o +obj-$(CONFIG_TEST_MIN_HEAP) += test_min_heap.o obj-$(CONFIG_TEST_LKM) += test_module.o obj-$(CONFIG_TEST_VMALLOC) += test_vmalloc.o obj-$(CONFIG_TEST_OVERFLOW) += test_overflow.o @@ -75,9 +83,9 @@ obj-$(CONFIG_TEST_USER_COPY) += test_user_copy.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_keys.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_key_base.o obj-$(CONFIG_TEST_PRINTF) += test_printf.o +obj-$(CONFIG_TEST_SCANF) += test_scanf.o obj-$(CONFIG_TEST_BITMAP) += test_bitmap.o obj-$(CONFIG_TEST_STRSCPY) += test_strscpy.o -obj-$(CONFIG_TEST_BITFIELD) += test_bitfield.o obj-$(CONFIG_TEST_UUID) += test_uuid.o obj-$(CONFIG_TEST_XARRAY) += test_xarray.o obj-$(CONFIG_TEST_PARMAN) += test_parman.o @@ -85,9 +93,39 @@ obj-$(CONFIG_TEST_KMOD) += test_kmod.o obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o obj-$(CONFIG_TEST_MEMCAT_P) += test_memcat_p.o obj-$(CONFIG_TEST_OBJAGG) += test_objagg.o +CFLAGS_test_stackinit.o += $(call cc-disable-warning, switch-unreachable) obj-$(CONFIG_TEST_STACKINIT) += test_stackinit.o obj-$(CONFIG_TEST_BLACKHOLE_DEV) += test_blackhole_dev.o obj-$(CONFIG_TEST_MEMINIT) += test_meminit.o +obj-$(CONFIG_TEST_LOCKUP) += test_lockup.o +obj-$(CONFIG_TEST_HMM) += test_hmm.o +obj-$(CONFIG_TEST_FREE_PAGES) += test_free_pages.o +obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o + +# +# CFLAGS for compiling floating point code inside the kernel. x86/Makefile turns +# off the generation of FPU/SSE* instructions for kernel proper but FPU_FLAGS +# get appended last to CFLAGS and thus override those previous compiler options. +# +FPU_CFLAGS := -msse -msse2 +ifdef CONFIG_CC_IS_GCC +# Stack alignment mismatch, proceed with caution. +# GCC < 7.1 cannot compile code using `double` and -mpreferred-stack-boundary=3 +# (8B stack alignment). +# See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53383 +# +# The "-msse" in the first argument is there so that the +# -mpreferred-stack-boundary=3 build error: +# +# -mpreferred-stack-boundary=3 is not between 4 and 12 +# +# can be triggered. Otherwise gcc doesn't complain. +FPU_CFLAGS += -mhard-float +FPU_CFLAGS += $(call cc-option,-msse -mpreferred-stack-boundary=3,-mpreferred-stack-boundary=4) +endif + +obj-$(CONFIG_TEST_FPU) += test_fpu.o +CFLAGS_test_fpu.o += $(FPU_CFLAGS) obj-$(CONFIG_TEST_LIVEPATCH) += livepatch/ @@ -111,6 +149,8 @@ obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o lib-y += logic_pio.o +lib-$(CONFIG_INDIRECT_IOMEM) += logic_iomem.o + obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o obj-$(CONFIG_BTREE) += btree.o @@ -121,6 +161,7 @@ obj-$(CONFIG_DEBUG_LIST) += list_debug.o obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o obj-$(CONFIG_BITREVERSE) += bitrev.o +obj-$(CONFIG_LINEAR_RANGES) += linear_ranges.o obj-$(CONFIG_PACKING) += packing.o obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o obj-$(CONFIG_CRC16) += crc16.o @@ -140,6 +181,7 @@ obj-$(CONFIG_842_COMPRESS) += 842/ obj-$(CONFIG_842_DECOMPRESS) += 842/ obj-$(CONFIG_ZLIB_INFLATE) += zlib_inflate/ obj-$(CONFIG_ZLIB_DEFLATE) += zlib_deflate/ +obj-$(CONFIG_ZLIB_DFLTCC) += zlib_dfltcc/ obj-$(CONFIG_REED_SOLOMON) += reed_solomon/ obj-$(CONFIG_BCH) += bch.o obj-$(CONFIG_LZO_COMPRESS) += lzo/ @@ -158,6 +200,7 @@ lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o lib-$(CONFIG_DECOMPRESS_XZ) += decompress_unxz.o lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o lib-$(CONFIG_DECOMPRESS_LZ4) += decompress_unlz4.o +lib-$(CONFIG_DECOMPRESS_ZSTD) += decompress_unzstd.o obj-$(CONFIG_TEXTSEARCH) += textsearch.o obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o @@ -169,6 +212,7 @@ obj-$(CONFIG_AUDIT_COMPAT_GENERIC) += compat_audit.o obj-$(CONFIG_IOMMU_HELPER) += iommu-helper.o obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o +obj-$(CONFIG_FAULT_INJECTION_USERCOPY) += fault-inject-usercopy.o obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o obj-$(CONFIG_PM_NOTIFIER_ERROR_INJECT) += pm-notifier-error-inject.o obj-$(CONFIG_NETDEV_NOTIFIER_ERROR_INJECT) += netdev-notifier-error-inject.o @@ -181,7 +225,7 @@ lib-$(CONFIG_GENERIC_BUG) += bug.o obj-$(CONFIG_HAVE_ARCH_TRACEHOOK) += syscall.o -obj-$(CONFIG_DYNAMIC_DEBUG) += dynamic_debug.o +obj-$(CONFIG_DYNAMIC_DEBUG_CORE) += dynamic_debug.o obj-$(CONFIG_SYMBOLIC_ERRNAME) += errname.o obj-$(CONFIG_NLATTR) += nlattr.o @@ -218,27 +262,34 @@ obj-$(CONFIG_MEMREGION) += memregion.o obj-$(CONFIG_STMP_DEVICE) += stmp_device.o obj-$(CONFIG_IRQ_POLL) += irq_poll.o +# stackdepot.c should not be instrumented or call instrumented functions. +# Prevent the compiler from calling builtins like memcmp() or bcmp() from this +# file. +CFLAGS_stackdepot.o += -fno-builtin obj-$(CONFIG_STACKDEPOT) += stackdepot.o KASAN_SANITIZE_stackdepot.o := n KCOV_INSTRUMENT_stackdepot.o := n libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \ - fdt_empty_tree.o + fdt_empty_tree.o fdt_addresses.o $(foreach file, $(libfdt_files), \ $(eval CFLAGS_$(file) = -I $(srctree)/scripts/dtc/libfdt)) lib-$(CONFIG_LIBFDT) += $(libfdt_files) +lib-$(CONFIG_BOOT_CONFIG) += bootconfig.o + obj-$(CONFIG_RBTREE_TEST) += rbtree_test.o obj-$(CONFIG_INTERVAL_TREE_TEST) += interval_tree_test.o obj-$(CONFIG_PERCPU_TEST) += percpu_test.o obj-$(CONFIG_ASN1) += asn1_decoder.o +obj-$(CONFIG_ASN1_ENCODER) += asn1_encoder.o obj-$(CONFIG_FONT_SUPPORT) += fonts/ -hostprogs-y := gen_crc32table -hostprogs-y += gen_crc64table +hostprogs := gen_crc32table +hostprogs += gen_crc64table clean-files := crc32table.h clean-files += crc64table.h @@ -275,11 +326,14 @@ quiet_cmd_build_OID_registry = GEN $@ clean-files += oid_registry_data.c obj-$(CONFIG_UCS2_STRING) += ucs2_string.o +ifneq ($(CONFIG_UBSAN_TRAP),y) obj-$(CONFIG_UBSAN) += ubsan.o +endif UBSAN_SANITIZE_ubsan.o := n KASAN_SANITIZE_ubsan.o := n -CFLAGS_ubsan.o := $(call cc-option, -fno-stack-protector) $(DISABLE_STACKLEAK_PLUGIN) +KCSAN_SANITIZE_ubsan.o := n +CFLAGS_ubsan.o := -fno-stack-protector $(DISABLE_STACKLEAK_PLUGIN) obj-$(CONFIG_SBITMAP) += sbitmap.o @@ -294,5 +348,50 @@ obj-$(CONFIG_GENERIC_LIB_CMPDI2) += cmpdi2.o obj-$(CONFIG_GENERIC_LIB_UCMPDI2) += ucmpdi2.o obj-$(CONFIG_OBJAGG) += objagg.o +# pldmfw library +obj-$(CONFIG_PLDMFW) += pldmfw/ + # KUnit tests +CFLAGS_bitfield_kunit.o := $(DISABLE_STRUCTLEAK_PLUGIN) +obj-$(CONFIG_BITFIELD_KUNIT) += bitfield_kunit.o obj-$(CONFIG_LIST_KUNIT_TEST) += list-test.o +obj-$(CONFIG_LINEAR_RANGES_TEST) += test_linear_ranges.o +obj-$(CONFIG_BITS_TEST) += test_bits.o +obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o +obj-$(CONFIG_SLUB_KUNIT_TEST) += slub_kunit.o +obj-$(CONFIG_MEMCPY_KUNIT_TEST) += memcpy_kunit.o + +obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o + +# FORTIFY_SOURCE compile-time behavior tests +TEST_FORTIFY_SRCS = $(wildcard $(srctree)/$(src)/test_fortify/*-*.c) +TEST_FORTIFY_LOGS = $(patsubst $(srctree)/$(src)/%.c, %.log, $(TEST_FORTIFY_SRCS)) +TEST_FORTIFY_LOG = test_fortify.log + +quiet_cmd_test_fortify = TEST $@ + cmd_test_fortify = $(CONFIG_SHELL) $(srctree)/scripts/test_fortify.sh \ + $< $@ "$(NM)" $(CC) $(c_flags) \ + $(call cc-disable-warning,fortify-source) + +targets += $(TEST_FORTIFY_LOGS) +clean-files += $(TEST_FORTIFY_LOGS) +clean-files += $(addsuffix .o, $(TEST_FORTIFY_LOGS)) +$(obj)/test_fortify/%.log: $(src)/test_fortify/%.c \ + $(src)/test_fortify/test_fortify.h \ + $(srctree)/include/linux/fortify-string.h \ + $(srctree)/scripts/test_fortify.sh \ + FORCE + $(call if_changed,test_fortify) + +quiet_cmd_gen_fortify_log = GEN $@ + cmd_gen_fortify_log = cat </dev/null $(filter-out FORCE,$^) 2>/dev/null > $@ || true + +targets += $(TEST_FORTIFY_LOG) +clean-files += $(TEST_FORTIFY_LOG) +$(obj)/$(TEST_FORTIFY_LOG): $(addprefix $(obj)/, $(TEST_FORTIFY_LOGS)) FORCE + $(call if_changed,gen_fortify_log) + +# Fake dependency to trigger the fortify tests. +ifeq ($(CONFIG_FORTIFY_SOURCE),y) +$(obj)/string.o: $(obj)/$(TEST_FORTIFY_LOG) +endif diff --git a/lib/asn1_decoder.c b/lib/asn1_decoder.c index 58f72b25f8e9..13da529e2e72 100644 --- a/lib/asn1_decoder.c +++ b/lib/asn1_decoder.c @@ -381,7 +381,7 @@ next_op: case ASN1_OP_END_SET_ACT: if (unlikely(!(flags & FLAG_MATCHED))) goto tag_mismatch; - /* fall through */ + fallthrough; case ASN1_OP_END_SEQ: case ASN1_OP_END_SET_OF: @@ -448,7 +448,7 @@ next_op: pc += asn1_op_lengths[op]; goto next_op; } - /* fall through */ + fallthrough; case ASN1_OP_ACT: ret = actions[machine[pc + 1]](context, hdr, tag, data + tdp, len); diff --git a/lib/asn1_encoder.c b/lib/asn1_encoder.c new file mode 100644 index 000000000000..27bbe891714f --- /dev/null +++ b/lib/asn1_encoder.c @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Simple encoder primitives for ASN.1 BER/DER/CER + * + * Copyright (C) 2019 James.Bottomley@HansenPartnership.com + */ + +#include <linux/asn1_encoder.h> +#include <linux/bug.h> +#include <linux/string.h> +#include <linux/module.h> + +/** + * asn1_encode_integer() - encode positive integer to ASN.1 + * @data: pointer to the pointer to the data + * @end_data: end of data pointer, points one beyond last usable byte in @data + * @integer: integer to be encoded + * + * This is a simplified encoder: it only currently does + * positive integers, but it should be simple enough to add the + * negative case if a use comes along. + */ +unsigned char * +asn1_encode_integer(unsigned char *data, const unsigned char *end_data, + s64 integer) +{ + int data_len = end_data - data; + unsigned char *d = &data[2]; + bool found = false; + int i; + + if (WARN(integer < 0, + "BUG: integer encode only supports positive integers")) + return ERR_PTR(-EINVAL); + + if (IS_ERR(data)) + return data; + + /* need at least 3 bytes for tag, length and integer encoding */ + if (data_len < 3) + return ERR_PTR(-EINVAL); + + /* remaining length where at d (the start of the integer encoding) */ + data_len -= 2; + + data[0] = _tag(UNIV, PRIM, INT); + if (integer == 0) { + *d++ = 0; + goto out; + } + + for (i = sizeof(integer); i > 0 ; i--) { + int byte = integer >> (8 * (i - 1)); + + if (!found && byte == 0) + continue; + + /* + * for a positive number the first byte must have bit + * 7 clear in two's complement (otherwise it's a + * negative number) so prepend a leading zero if + * that's not the case + */ + if (!found && (byte & 0x80)) { + /* + * no check needed here, we already know we + * have len >= 1 + */ + *d++ = 0; + data_len--; + } + + found = true; + if (data_len == 0) + return ERR_PTR(-EINVAL); + + *d++ = byte; + data_len--; + } + + out: + data[1] = d - data - 2; + + return d; +} +EXPORT_SYMBOL_GPL(asn1_encode_integer); + +/* calculate the base 128 digit values setting the top bit of the first octet */ +static int asn1_encode_oid_digit(unsigned char **_data, int *data_len, u32 oid) +{ + unsigned char *data = *_data; + int start = 7 + 7 + 7 + 7; + int ret = 0; + + if (*data_len < 1) + return -EINVAL; + + /* quick case */ + if (oid == 0) { + *data++ = 0x80; + (*data_len)--; + goto out; + } + + while (oid >> start == 0) + start -= 7; + + while (start > 0 && *data_len > 0) { + u8 byte; + + byte = oid >> start; + oid = oid - (byte << start); + start -= 7; + byte |= 0x80; + *data++ = byte; + (*data_len)--; + } + + if (*data_len > 0) { + *data++ = oid; + (*data_len)--; + } else { + ret = -EINVAL; + } + + out: + *_data = data; + return ret; +} + +/** + * asn1_encode_oid() - encode an oid to ASN.1 + * @data: position to begin encoding at + * @end_data: end of data pointer, points one beyond last usable byte in @data + * @oid: array of oids + * @oid_len: length of oid array + * + * this encodes an OID up to ASN.1 when presented as an array of OID values + */ +unsigned char * +asn1_encode_oid(unsigned char *data, const unsigned char *end_data, + u32 oid[], int oid_len) +{ + int data_len = end_data - data; + unsigned char *d = data + 2; + int i, ret; + + if (WARN(oid_len < 2, "OID must have at least two elements")) + return ERR_PTR(-EINVAL); + + if (WARN(oid_len > 32, "OID is too large")) + return ERR_PTR(-EINVAL); + + if (IS_ERR(data)) + return data; + + + /* need at least 3 bytes for tag, length and OID encoding */ + if (data_len < 3) + return ERR_PTR(-EINVAL); + + data[0] = _tag(UNIV, PRIM, OID); + *d++ = oid[0] * 40 + oid[1]; + + data_len -= 3; + + ret = 0; + + for (i = 2; i < oid_len; i++) { + ret = asn1_encode_oid_digit(&d, &data_len, oid[i]); + if (ret < 0) + return ERR_PTR(ret); + } + + data[1] = d - data - 2; + + return d; +} +EXPORT_SYMBOL_GPL(asn1_encode_oid); + +/** + * asn1_encode_length() - encode a length to follow an ASN.1 tag + * @data: pointer to encode at + * @data_len: pointer to remaining length (adjusted by routine) + * @len: length to encode + * + * This routine can encode lengths up to 65535 using the ASN.1 rules. + * It will accept a negative length and place a zero length tag + * instead (to keep the ASN.1 valid). This convention allows other + * encoder primitives to accept negative lengths as singalling the + * sequence will be re-encoded when the length is known. + */ +static int asn1_encode_length(unsigned char **data, int *data_len, int len) +{ + if (*data_len < 1) + return -EINVAL; + + if (len < 0) { + *((*data)++) = 0; + (*data_len)--; + return 0; + } + + if (len <= 0x7f) { + *((*data)++) = len; + (*data_len)--; + return 0; + } + + if (*data_len < 2) + return -EINVAL; + + if (len <= 0xff) { + *((*data)++) = 0x81; + *((*data)++) = len & 0xff; + *data_len -= 2; + return 0; + } + + if (*data_len < 3) + return -EINVAL; + + if (len <= 0xffff) { + *((*data)++) = 0x82; + *((*data)++) = (len >> 8) & 0xff; + *((*data)++) = len & 0xff; + *data_len -= 3; + return 0; + } + + if (WARN(len > 0xffffff, "ASN.1 length can't be > 0xffffff")) + return -EINVAL; + + if (*data_len < 4) + return -EINVAL; + *((*data)++) = 0x83; + *((*data)++) = (len >> 16) & 0xff; + *((*data)++) = (len >> 8) & 0xff; + *((*data)++) = len & 0xff; + *data_len -= 4; + + return 0; +} + +/** + * asn1_encode_tag() - add a tag for optional or explicit value + * @data: pointer to place tag at + * @end_data: end of data pointer, points one beyond last usable byte in @data + * @tag: tag to be placed + * @string: the data to be tagged + * @len: the length of the data to be tagged + * + * Note this currently only handles short form tags < 31. + * + * Standard usage is to pass in a @tag, @string and @length and the + * @string will be ASN.1 encoded with @tag and placed into @data. If + * the encoding would put data past @end_data then an error is + * returned, otherwise a pointer to a position one beyond the encoding + * is returned. + * + * To encode in place pass a NULL @string and -1 for @len and the + * maximum allowable beginning and end of the data; all this will do + * is add the current maximum length and update the data pointer to + * the place where the tag contents should be placed is returned. The + * data should be copied in by the calling routine which should then + * repeat the prior statement but now with the known length. In order + * to avoid having to keep both before and after pointers, the repeat + * expects to be called with @data pointing to where the first encode + * returned it and still NULL for @string but the real length in @len. + */ +unsigned char * +asn1_encode_tag(unsigned char *data, const unsigned char *end_data, + u32 tag, const unsigned char *string, int len) +{ + int data_len = end_data - data; + int ret; + + if (WARN(tag > 30, "ASN.1 tag can't be > 30")) + return ERR_PTR(-EINVAL); + + if (!string && WARN(len > 127, + "BUG: recode tag is too big (>127)")) + return ERR_PTR(-EINVAL); + + if (IS_ERR(data)) + return data; + + if (!string && len > 0) { + /* + * we're recoding, so move back to the start of the + * tag and install a dummy length because the real + * data_len should be NULL + */ + data -= 2; + data_len = 2; + } + + if (data_len < 2) + return ERR_PTR(-EINVAL); + + *(data++) = _tagn(CONT, CONS, tag); + data_len--; + ret = asn1_encode_length(&data, &data_len, len); + if (ret < 0) + return ERR_PTR(ret); + + if (!string) + return data; + + if (data_len < len) + return ERR_PTR(-EINVAL); + + memcpy(data, string, len); + data += len; + + return data; +} +EXPORT_SYMBOL_GPL(asn1_encode_tag); + +/** + * asn1_encode_octet_string() - encode an ASN.1 OCTET STRING + * @data: pointer to encode at + * @end_data: end of data pointer, points one beyond last usable byte in @data + * @string: string to be encoded + * @len: length of string + * + * Note ASN.1 octet strings may contain zeros, so the length is obligatory. + */ +unsigned char * +asn1_encode_octet_string(unsigned char *data, + const unsigned char *end_data, + const unsigned char *string, u32 len) +{ + int data_len = end_data - data; + int ret; + + if (IS_ERR(data)) + return data; + + /* need minimum of 2 bytes for tag and length of zero length string */ + if (data_len < 2) + return ERR_PTR(-EINVAL); + + *(data++) = _tag(UNIV, PRIM, OTS); + data_len--; + + ret = asn1_encode_length(&data, &data_len, len); + if (ret) + return ERR_PTR(ret); + + if (data_len < len) + return ERR_PTR(-EINVAL); + + memcpy(data, string, len); + data += len; + + return data; +} +EXPORT_SYMBOL_GPL(asn1_encode_octet_string); + +/** + * asn1_encode_sequence() - wrap a byte stream in an ASN.1 SEQUENCE + * @data: pointer to encode at + * @end_data: end of data pointer, points one beyond last usable byte in @data + * @seq: data to be encoded as a sequence + * @len: length of the data to be encoded as a sequence + * + * Fill in a sequence. To encode in place, pass NULL for @seq and -1 + * for @len; then call again once the length is known (still with NULL + * for @seq). In order to avoid having to keep both before and after + * pointers, the repeat expects to be called with @data pointing to + * where the first encode placed it. + */ +unsigned char * +asn1_encode_sequence(unsigned char *data, const unsigned char *end_data, + const unsigned char *seq, int len) +{ + int data_len = end_data - data; + int ret; + + if (!seq && WARN(len > 127, + "BUG: recode sequence is too big (>127)")) + return ERR_PTR(-EINVAL); + + if (IS_ERR(data)) + return data; + + if (!seq && len >= 0) { + /* + * we're recoding, so move back to the start of the + * sequence and install a dummy length because the + * real length should be NULL + */ + data -= 2; + data_len = 2; + } + + if (data_len < 2) + return ERR_PTR(-EINVAL); + + *(data++) = _tag(UNIV, CONS, SEQ); + data_len--; + + ret = asn1_encode_length(&data, &data_len, len); + if (ret) + return ERR_PTR(ret); + + if (!seq) + return data; + + if (data_len < len) + return ERR_PTR(-EINVAL); + + memcpy(data, seq, len); + data += len; + + return data; +} +EXPORT_SYMBOL_GPL(asn1_encode_sequence); + +/** + * asn1_encode_boolean() - encode a boolean value to ASN.1 + * @data: pointer to encode at + * @end_data: end of data pointer, points one beyond last usable byte in @data + * @val: the boolean true/false value + */ +unsigned char * +asn1_encode_boolean(unsigned char *data, const unsigned char *end_data, + bool val) +{ + int data_len = end_data - data; + + if (IS_ERR(data)) + return data; + + /* booleans are 3 bytes: tag, length == 1 and value == 0 or 1 */ + if (data_len < 3) + return ERR_PTR(-EINVAL); + + *(data++) = _tag(UNIV, PRIM, BOOL); + data_len--; + + asn1_encode_length(&data, &data_len, 1); + + if (val) + *(data++) = 1; + else + *(data++) = 0; + + return data; +} +EXPORT_SYMBOL_GPL(asn1_encode_boolean); + +MODULE_LICENSE("GPL"); diff --git a/lib/assoc_array.c b/lib/assoc_array.c index 6f4bcf524554..079c72e26493 100644 --- a/lib/assoc_array.c +++ b/lib/assoc_array.c @@ -741,8 +741,7 @@ all_leaves_cluster_together: keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE); keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT; - new_s0 = kzalloc(sizeof(struct assoc_array_shortcut) + - keylen * sizeof(unsigned long), GFP_KERNEL); + new_s0 = kzalloc(struct_size(new_s0, index_key, keylen), GFP_KERNEL); if (!new_s0) return false; edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s0); @@ -849,8 +848,8 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE); keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT; - new_s0 = kzalloc(sizeof(struct assoc_array_shortcut) + - keylen * sizeof(unsigned long), GFP_KERNEL); + new_s0 = kzalloc(struct_size(new_s0, index_key, keylen), + GFP_KERNEL); if (!new_s0) return false; edit->new_meta[1] = assoc_array_shortcut_to_ptr(new_s0); @@ -864,7 +863,7 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, new_n0->parent_slot = 0; memcpy(new_s0->index_key, shortcut->index_key, - keylen * sizeof(unsigned long)); + flex_array_size(new_s0, index_key, keylen)); blank = ULONG_MAX << (diff & ASSOC_ARRAY_KEY_CHUNK_MASK); pr_devel("blank off [%zu] %d: %lx\n", keylen - 1, diff, blank); @@ -899,8 +898,8 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE); keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT; - new_s1 = kzalloc(sizeof(struct assoc_array_shortcut) + - keylen * sizeof(unsigned long), GFP_KERNEL); + new_s1 = kzalloc(struct_size(new_s1, index_key, keylen), + GFP_KERNEL); if (!new_s1) return false; edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s1); @@ -913,7 +912,7 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, new_n0->slots[sc_slot] = assoc_array_shortcut_to_ptr(new_s1); memcpy(new_s1->index_key, shortcut->index_key, - keylen * sizeof(unsigned long)); + flex_array_size(new_s1, index_key, keylen)); edit->set[1].ptr = &side->back_pointer; edit->set[1].to = assoc_array_shortcut_to_ptr(new_s1); @@ -1113,7 +1112,7 @@ struct assoc_array_edit *assoc_array_delete(struct assoc_array *array, index_key)) goto found_leaf; } - /* fall through */ + fallthrough; case assoc_array_walk_tree_empty: case assoc_array_walk_found_wrong_shortcut: default: @@ -1490,13 +1489,12 @@ descend: shortcut = assoc_array_ptr_to_shortcut(cursor); keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE); keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT; - new_s = kmalloc(sizeof(struct assoc_array_shortcut) + - keylen * sizeof(unsigned long), GFP_KERNEL); + new_s = kmalloc(struct_size(new_s, index_key, keylen), + GFP_KERNEL); if (!new_s) goto enomem; pr_devel("dup shortcut %p -> %p\n", shortcut, new_s); - memcpy(new_s, shortcut, (sizeof(struct assoc_array_shortcut) + - keylen * sizeof(unsigned long))); + memcpy(new_s, shortcut, struct_size(new_s, index_key, keylen)); new_s->back_pointer = new_parent; new_s->parent_slot = shortcut->parent_slot; *new_ptr_pp = new_parent = assoc_array_shortcut_to_ptr(new_s); diff --git a/lib/atomic64.c b/lib/atomic64.c index e98c85a99787..3df653994177 100644 --- a/lib/atomic64.c +++ b/lib/atomic64.c @@ -42,7 +42,7 @@ static inline raw_spinlock_t *lock_addr(const atomic64_t *v) return &atomic64_lock[addr & (NR_LOCKS - 1)].lock; } -s64 atomic64_read(const atomic64_t *v) +s64 generic_atomic64_read(const atomic64_t *v) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -53,9 +53,9 @@ s64 atomic64_read(const atomic64_t *v) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_read); +EXPORT_SYMBOL(generic_atomic64_read); -void atomic64_set(atomic64_t *v, s64 i) +void generic_atomic64_set(atomic64_t *v, s64 i) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -64,10 +64,10 @@ void atomic64_set(atomic64_t *v, s64 i) v->counter = i; raw_spin_unlock_irqrestore(lock, flags); } -EXPORT_SYMBOL(atomic64_set); +EXPORT_SYMBOL(generic_atomic64_set); #define ATOMIC64_OP(op, c_op) \ -void atomic64_##op(s64 a, atomic64_t *v) \ +void generic_atomic64_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ @@ -76,10 +76,10 @@ void atomic64_##op(s64 a, atomic64_t *v) \ v->counter c_op a; \ raw_spin_unlock_irqrestore(lock, flags); \ } \ -EXPORT_SYMBOL(atomic64_##op); +EXPORT_SYMBOL(generic_atomic64_##op); #define ATOMIC64_OP_RETURN(op, c_op) \ -s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ +s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ @@ -90,10 +90,10 @@ s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ raw_spin_unlock_irqrestore(lock, flags); \ return val; \ } \ -EXPORT_SYMBOL(atomic64_##op##_return); +EXPORT_SYMBOL(generic_atomic64_##op##_return); #define ATOMIC64_FETCH_OP(op, c_op) \ -s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ +s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ @@ -105,7 +105,7 @@ s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ raw_spin_unlock_irqrestore(lock, flags); \ return val; \ } \ -EXPORT_SYMBOL(atomic64_fetch_##op); +EXPORT_SYMBOL(generic_atomic64_fetch_##op); #define ATOMIC64_OPS(op, c_op) \ ATOMIC64_OP(op, c_op) \ @@ -130,7 +130,7 @@ ATOMIC64_OPS(xor, ^=) #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -s64 atomic64_dec_if_positive(atomic64_t *v) +s64 generic_atomic64_dec_if_positive(atomic64_t *v) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -143,9 +143,9 @@ s64 atomic64_dec_if_positive(atomic64_t *v) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_dec_if_positive); +EXPORT_SYMBOL(generic_atomic64_dec_if_positive); -s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) +s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -158,9 +158,9 @@ s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_cmpxchg); +EXPORT_SYMBOL(generic_atomic64_cmpxchg); -s64 atomic64_xchg(atomic64_t *v, s64 new) +s64 generic_atomic64_xchg(atomic64_t *v, s64 new) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -172,9 +172,9 @@ s64 atomic64_xchg(atomic64_t *v, s64 new) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_xchg); +EXPORT_SYMBOL(generic_atomic64_xchg); -s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -188,4 +188,4 @@ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) return val; } -EXPORT_SYMBOL(atomic64_fetch_add_unless); +EXPORT_SYMBOL(generic_atomic64_fetch_add_unless); diff --git a/lib/audit.c b/lib/audit.c index 5004bff928a7..738bda22dd39 100644 --- a/lib/audit.c +++ b/lib/audit.c @@ -45,23 +45,27 @@ int audit_classify_syscall(int abi, unsigned syscall) switch(syscall) { #ifdef __NR_open case __NR_open: - return 2; + return AUDITSC_OPEN; #endif #ifdef __NR_openat case __NR_openat: - return 3; + return AUDITSC_OPENAT; #endif #ifdef __NR_socketcall case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; #endif #ifdef __NR_execveat case __NR_execveat: #endif case __NR_execve: - return 5; + return AUDITSC_EXECVE; +#ifdef __NR_openat2 + case __NR_openat2: + return AUDITSC_OPENAT2; +#endif default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/lib/bch.c b/lib/bch.c index 5db6d3a4c8a6..c8095f30f254 100644 --- a/lib/bch.c +++ b/lib/bch.c @@ -23,15 +23,15 @@ * This library provides runtime configurable encoding/decoding of binary * Bose-Chaudhuri-Hocquenghem (BCH) codes. * - * Call init_bch to get a pointer to a newly allocated bch_control structure for + * Call bch_init to get a pointer to a newly allocated bch_control structure for * the given m (Galois field order), t (error correction capability) and * (optional) primitive polynomial parameters. * - * Call encode_bch to compute and store ecc parity bytes to a given buffer. - * Call decode_bch to detect and locate errors in received data. + * Call bch_encode to compute and store ecc parity bytes to a given buffer. + * Call bch_decode to detect and locate errors in received data. * * On systems supporting hw BCH features, intermediate results may be provided - * to decode_bch in order to skip certain steps. See decode_bch() documentation + * to bch_decode in order to skip certain steps. See bch_decode() documentation * for details. * * Option CONFIG_BCH_CONST_PARAMS can be used to force fixed values of @@ -102,7 +102,7 @@ */ struct gf_poly { unsigned int deg; /* polynomial degree */ - unsigned int c[0]; /* polynomial terms */ + unsigned int c[]; /* polynomial terms */ }; /* given its degree, compute a polynomial size in bytes */ @@ -114,10 +114,53 @@ struct gf_poly_deg1 { unsigned int c[2]; }; +static u8 swap_bits_table[] = { + 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, + 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, + 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, + 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, + 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, + 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, + 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, + 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, + 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, + 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, + 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, + 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, + 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, + 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, + 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, + 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, + 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, + 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, + 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, + 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, + 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, + 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, + 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, + 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, + 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, + 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, + 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, + 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, + 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, + 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, + 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, + 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, +}; + +static u8 swap_bits(struct bch_control *bch, u8 in) +{ + if (!bch->swap_bits) + return in; + + return swap_bits_table[in]; +} + /* - * same as encode_bch(), but process input data one byte at a time + * same as bch_encode(), but process input data one byte at a time */ -static void encode_bch_unaligned(struct bch_control *bch, +static void bch_encode_unaligned(struct bch_control *bch, const unsigned char *data, unsigned int len, uint32_t *ecc) { @@ -126,7 +169,9 @@ static void encode_bch_unaligned(struct bch_control *bch, const int l = BCH_ECC_WORDS(bch)-1; while (len--) { - p = bch->mod8_tab + (l+1)*(((ecc[0] >> 24)^(*data++)) & 0xff); + u8 tmp = swap_bits(bch, *data++); + + p = bch->mod8_tab + (l+1)*(((ecc[0] >> 24)^(tmp)) & 0xff); for (i = 0; i < l; i++) ecc[i] = ((ecc[i] << 8)|(ecc[i+1] >> 24))^(*p++); @@ -145,10 +190,16 @@ static void load_ecc8(struct bch_control *bch, uint32_t *dst, unsigned int i, nwords = BCH_ECC_WORDS(bch)-1; for (i = 0; i < nwords; i++, src += 4) - dst[i] = (src[0] << 24)|(src[1] << 16)|(src[2] << 8)|src[3]; + dst[i] = ((u32)swap_bits(bch, src[0]) << 24) | + ((u32)swap_bits(bch, src[1]) << 16) | + ((u32)swap_bits(bch, src[2]) << 8) | + swap_bits(bch, src[3]); memcpy(pad, src, BCH_ECC_BYTES(bch)-4*nwords); - dst[nwords] = (pad[0] << 24)|(pad[1] << 16)|(pad[2] << 8)|pad[3]; + dst[nwords] = ((u32)swap_bits(bch, pad[0]) << 24) | + ((u32)swap_bits(bch, pad[1]) << 16) | + ((u32)swap_bits(bch, pad[2]) << 8) | + swap_bits(bch, pad[3]); } /* @@ -161,20 +212,20 @@ static void store_ecc8(struct bch_control *bch, uint8_t *dst, unsigned int i, nwords = BCH_ECC_WORDS(bch)-1; for (i = 0; i < nwords; i++) { - *dst++ = (src[i] >> 24); - *dst++ = (src[i] >> 16) & 0xff; - *dst++ = (src[i] >> 8) & 0xff; - *dst++ = (src[i] >> 0) & 0xff; + *dst++ = swap_bits(bch, src[i] >> 24); + *dst++ = swap_bits(bch, src[i] >> 16); + *dst++ = swap_bits(bch, src[i] >> 8); + *dst++ = swap_bits(bch, src[i]); } - pad[0] = (src[nwords] >> 24); - pad[1] = (src[nwords] >> 16) & 0xff; - pad[2] = (src[nwords] >> 8) & 0xff; - pad[3] = (src[nwords] >> 0) & 0xff; + pad[0] = swap_bits(bch, src[nwords] >> 24); + pad[1] = swap_bits(bch, src[nwords] >> 16); + pad[2] = swap_bits(bch, src[nwords] >> 8); + pad[3] = swap_bits(bch, src[nwords]); memcpy(dst, pad, BCH_ECC_BYTES(bch)-4*nwords); } /** - * encode_bch - calculate BCH ecc parity of data + * bch_encode - calculate BCH ecc parity of data * @bch: BCH control structure * @data: data to encode * @len: data length in bytes @@ -187,7 +238,7 @@ static void store_ecc8(struct bch_control *bch, uint8_t *dst, * The exact number of computed ecc parity bits is given by member @ecc_bits of * @bch; it may be less than m*t for large values of t. */ -void encode_bch(struct bch_control *bch, const uint8_t *data, +void bch_encode(struct bch_control *bch, const uint8_t *data, unsigned int len, uint8_t *ecc) { const unsigned int l = BCH_ECC_WORDS(bch)-1; @@ -215,7 +266,7 @@ void encode_bch(struct bch_control *bch, const uint8_t *data, m = ((unsigned long)data) & 3; if (m) { mlen = (len < (4-m)) ? len : 4-m; - encode_bch_unaligned(bch, data, mlen, bch->ecc_buf); + bch_encode_unaligned(bch, data, mlen, bch->ecc_buf); data += mlen; len -= mlen; } @@ -240,7 +291,13 @@ void encode_bch(struct bch_control *bch, const uint8_t *data, */ while (mlen--) { /* input data is read in big-endian format */ - w = r[0]^cpu_to_be32(*pdata++); + w = cpu_to_be32(*pdata++); + if (bch->swap_bits) + w = (u32)swap_bits(bch, w) | + ((u32)swap_bits(bch, w >> 8) << 8) | + ((u32)swap_bits(bch, w >> 16) << 16) | + ((u32)swap_bits(bch, w >> 24) << 24); + w ^= r[0]; p0 = tab0 + (l+1)*((w >> 0) & 0xff); p1 = tab1 + (l+1)*((w >> 8) & 0xff); p2 = tab2 + (l+1)*((w >> 16) & 0xff); @@ -255,13 +312,13 @@ void encode_bch(struct bch_control *bch, const uint8_t *data, /* process last unaligned bytes */ if (len) - encode_bch_unaligned(bch, data, len, bch->ecc_buf); + bch_encode_unaligned(bch, data, len, bch->ecc_buf); /* store ecc parity bytes into original parity buffer */ if (ecc) store_ecc8(bch, ecc, bch->ecc_buf); } -EXPORT_SYMBOL_GPL(encode_bch); +EXPORT_SYMBOL_GPL(bch_encode); static inline int modulo(struct bch_control *bch, unsigned int v) { @@ -527,7 +584,7 @@ static int find_affine4_roots(struct bch_control *bch, unsigned int a, k = a_log(bch, a); rows[0] = c; - /* buid linear system to solve X^4+aX^2+bX+c = 0 */ + /* build linear system to solve X^4+aX^2+bX+c = 0 */ for (i = 0; i < m; i++) { rows[i+1] = bch->a_pow_tab[4*i]^ (a ? bch->a_pow_tab[mod_s(bch, k)] : 0)^ @@ -952,7 +1009,7 @@ static int chien_search(struct bch_control *bch, unsigned int len, #endif /* USE_CHIEN_SEARCH */ /** - * decode_bch - decode received codeword and find bit error locations + * bch_decode - decode received codeword and find bit error locations * @bch: BCH control structure * @data: received data, ignored if @calc_ecc is provided * @len: data length in bytes, must always be provided @@ -966,22 +1023,22 @@ static int chien_search(struct bch_control *bch, unsigned int len, * invalid parameters were provided * * Depending on the available hw BCH support and the need to compute @calc_ecc - * separately (using encode_bch()), this function should be called with one of + * separately (using bch_encode()), this function should be called with one of * the following parameter configurations - * * by providing @data and @recv_ecc only: - * decode_bch(@bch, @data, @len, @recv_ecc, NULL, NULL, @errloc) + * bch_decode(@bch, @data, @len, @recv_ecc, NULL, NULL, @errloc) * * by providing @recv_ecc and @calc_ecc: - * decode_bch(@bch, NULL, @len, @recv_ecc, @calc_ecc, NULL, @errloc) + * bch_decode(@bch, NULL, @len, @recv_ecc, @calc_ecc, NULL, @errloc) * * by providing ecc = recv_ecc XOR calc_ecc: - * decode_bch(@bch, NULL, @len, NULL, ecc, NULL, @errloc) + * bch_decode(@bch, NULL, @len, NULL, ecc, NULL, @errloc) * * by providing syndrome results @syn: - * decode_bch(@bch, NULL, @len, NULL, NULL, @syn, @errloc) + * bch_decode(@bch, NULL, @len, NULL, NULL, @syn, @errloc) * - * Once decode_bch() has successfully returned with a positive value, error + * Once bch_decode() has successfully returned with a positive value, error * locations returned in array @errloc should be interpreted as follows - * * if (errloc[n] >= 8*len), then n-th error is located in ecc (no need for @@ -993,7 +1050,7 @@ static int chien_search(struct bch_control *bch, unsigned int len, * Note that this function does not perform any data correction by itself, it * merely indicates error locations. */ -int decode_bch(struct bch_control *bch, const uint8_t *data, unsigned int len, +int bch_decode(struct bch_control *bch, const uint8_t *data, unsigned int len, const uint8_t *recv_ecc, const uint8_t *calc_ecc, const unsigned int *syn, unsigned int *errloc) { @@ -1012,7 +1069,7 @@ int decode_bch(struct bch_control *bch, const uint8_t *data, unsigned int len, /* compute received data ecc into an internal buffer */ if (!data || !recv_ecc) return -EINVAL; - encode_bch(bch, data, len, NULL); + bch_encode(bch, data, len, NULL); } else { /* load provided calculated ecc */ load_ecc8(bch, bch->ecc_buf, calc_ecc); @@ -1048,12 +1105,14 @@ int decode_bch(struct bch_control *bch, const uint8_t *data, unsigned int len, break; } errloc[i] = nbits-1-errloc[i]; - errloc[i] = (errloc[i] & ~7)|(7-(errloc[i] & 7)); + if (!bch->swap_bits) + errloc[i] = (errloc[i] & ~7) | + (7-(errloc[i] & 7)); } } return (err >= 0) ? err : -EBADMSG; } -EXPORT_SYMBOL_GPL(decode_bch); +EXPORT_SYMBOL_GPL(bch_decode); /* * generate Galois field lookup tables @@ -1236,27 +1295,29 @@ finish: } /** - * init_bch - initialize a BCH encoder/decoder + * bch_init - initialize a BCH encoder/decoder * @m: Galois field order, should be in the range 5-15 * @t: maximum error correction capability, in bits * @prim_poly: user-provided primitive polynomial (or 0 to use default) + * @swap_bits: swap bits within data and syndrome bytes * * Returns: * a newly allocated BCH control structure if successful, NULL otherwise * * This initialization can take some time, as lookup tables are built for fast * encoding/decoding; make sure not to call this function from a time critical - * path. Usually, init_bch() should be called on module/driver init and - * free_bch() should be called to release memory on exit. + * path. Usually, bch_init() should be called on module/driver init and + * bch_free() should be called to release memory on exit. * * You may provide your own primitive polynomial of degree @m in argument - * @prim_poly, or let init_bch() use its default polynomial. + * @prim_poly, or let bch_init() use its default polynomial. * - * Once init_bch() has successfully returned a pointer to a newly allocated + * Once bch_init() has successfully returned a pointer to a newly allocated * BCH control structure, ecc length in bytes is given by member @ecc_bytes of * the structure. */ -struct bch_control *init_bch(int m, int t, unsigned int prim_poly) +struct bch_control *bch_init(int m, int t, unsigned int prim_poly, + bool swap_bits) { int err = 0; unsigned int i, words; @@ -1321,6 +1382,7 @@ struct bch_control *init_bch(int m, int t, unsigned int prim_poly) bch->syn = bch_alloc(2*t*sizeof(*bch->syn), &err); bch->cache = bch_alloc(2*t*sizeof(*bch->cache), &err); bch->elp = bch_alloc((t+1)*sizeof(struct gf_poly_deg1), &err); + bch->swap_bits = swap_bits; for (i = 0; i < ARRAY_SIZE(bch->poly_2t); i++) bch->poly_2t[i] = bch_alloc(GF_POLY_SZ(2*t), &err); @@ -1347,16 +1409,16 @@ struct bch_control *init_bch(int m, int t, unsigned int prim_poly) return bch; fail: - free_bch(bch); + bch_free(bch); return NULL; } -EXPORT_SYMBOL_GPL(init_bch); +EXPORT_SYMBOL_GPL(bch_init); /** - * free_bch - free the BCH control structure + * bch_free - free the BCH control structure * @bch: BCH control structure to release */ -void free_bch(struct bch_control *bch) +void bch_free(struct bch_control *bch) { unsigned int i; @@ -1377,7 +1439,7 @@ void free_bch(struct bch_control *bch) kfree(bch); } } -EXPORT_SYMBOL_GPL(free_bch); +EXPORT_SYMBOL_GPL(bch_free); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Ivan Djelic <ivan.djelic@parrot.com>"); diff --git a/lib/test_bitfield.c b/lib/bitfield_kunit.c index 5b8f4108662d..1473d8b4bf0f 100644 --- a/lib/test_bitfield.c +++ b/lib/bitfield_kunit.c @@ -5,8 +5,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/kernel.h> -#include <linux/module.h> +#include <kunit/test.h> #include <linux/bitfield.h> #define CHECK_ENC_GET_U(tp, v, field, res) do { \ @@ -14,13 +13,11 @@ u##tp _res; \ \ _res = u##tp##_encode_bits(v, field); \ - if (_res != res) { \ - pr_warn("u" #tp "_encode_bits(" #v ", " #field ") is 0x%llx != " #res "\n",\ - (u64)_res); \ - return -EINVAL; \ - } \ - if (u##tp##_get_bits(_res, field) != v) \ - return -EINVAL; \ + KUNIT_ASSERT_FALSE_MSG(context, _res != res, \ + "u" #tp "_encode_bits(" #v ", " #field ") is 0x%llx != " #res "\n", \ + (u64)_res); \ + KUNIT_ASSERT_FALSE(context, \ + u##tp##_get_bits(_res, field) != v); \ } \ } while (0) @@ -29,14 +26,13 @@ __le##tp _res; \ \ _res = le##tp##_encode_bits(v, field); \ - if (_res != cpu_to_le##tp(res)) { \ - pr_warn("le" #tp "_encode_bits(" #v ", " #field ") is 0x%llx != 0x%llx\n",\ - (u64)le##tp##_to_cpu(_res), \ - (u64)(res)); \ - return -EINVAL; \ - } \ - if (le##tp##_get_bits(_res, field) != v) \ - return -EINVAL; \ + KUNIT_ASSERT_FALSE_MSG(context, \ + _res != cpu_to_le##tp(res), \ + "le" #tp "_encode_bits(" #v ", " #field ") is 0x%llx != 0x%llx",\ + (u64)le##tp##_to_cpu(_res), \ + (u64)(res)); \ + KUNIT_ASSERT_FALSE(context, \ + le##tp##_get_bits(_res, field) != v);\ } \ } while (0) @@ -45,14 +41,13 @@ __be##tp _res; \ \ _res = be##tp##_encode_bits(v, field); \ - if (_res != cpu_to_be##tp(res)) { \ - pr_warn("be" #tp "_encode_bits(" #v ", " #field ") is 0x%llx != 0x%llx\n",\ - (u64)be##tp##_to_cpu(_res), \ - (u64)(res)); \ - return -EINVAL; \ - } \ - if (be##tp##_get_bits(_res, field) != v) \ - return -EINVAL; \ + KUNIT_ASSERT_FALSE_MSG(context, \ + _res != cpu_to_be##tp(res), \ + "be" #tp "_encode_bits(" #v ", " #field ") is 0x%llx != 0x%llx", \ + (u64)be##tp##_to_cpu(_res), \ + (u64)(res)); \ + KUNIT_ASSERT_FALSE(context, \ + be##tp##_get_bits(_res, field) != v);\ } \ } while (0) @@ -62,7 +57,7 @@ CHECK_ENC_GET_BE(tp, v, field, res); \ } while (0) -static int test_constants(void) +static void __init test_bitfields_constants(struct kunit *context) { /* * NOTE @@ -95,19 +90,17 @@ static int test_constants(void) CHECK_ENC_GET(64, 7, 0x00f0000000000000ull, 0x0070000000000000ull); CHECK_ENC_GET(64, 14, 0x0f00000000000000ull, 0x0e00000000000000ull); CHECK_ENC_GET(64, 15, 0xf000000000000000ull, 0xf000000000000000ull); - - return 0; } #define CHECK(tp, mask) do { \ u64 v; \ \ for (v = 0; v < 1 << hweight32(mask); v++) \ - if (tp##_encode_bits(v, mask) != v << __ffs64(mask)) \ - return -EINVAL; \ + KUNIT_ASSERT_FALSE(context, \ + tp##_encode_bits(v, mask) != v << __ffs64(mask));\ } while (0) -static int test_variables(void) +static void __init test_bitfields_variables(struct kunit *context) { CHECK(u8, 0x0f); CHECK(u8, 0xf0); @@ -130,39 +123,32 @@ static int test_variables(void) CHECK(u64, 0x000000007f000000ull); CHECK(u64, 0x0000000018000000ull); CHECK(u64, 0x0000001f8000000ull); - - return 0; } -static int __init test_bitfields(void) -{ - int ret = test_constants(); - - if (ret) { - pr_warn("constant tests failed!\n"); - return ret; - } - - ret = test_variables(); - if (ret) { - pr_warn("variable tests failed!\n"); - return ret; - } - #ifdef TEST_BITFIELD_COMPILE +static void __init test_bitfields_compile(struct kunit *context) +{ /* these should fail compilation */ CHECK_ENC_GET(16, 16, 0x0f00, 0x1000); u32_encode_bits(7, 0x06000000); /* this should at least give a warning */ u16_encode_bits(0, 0x60000); +} #endif - pr_info("tests passed\n"); +static struct kunit_case __refdata bitfields_test_cases[] = { + KUNIT_CASE(test_bitfields_constants), + KUNIT_CASE(test_bitfields_variables), + {} +}; - return 0; -} -module_init(test_bitfields) +static struct kunit_suite bitfields_test_suite = { + .name = "bitfields", + .test_cases = bitfields_test_cases, +}; + +kunit_test_suites(&bitfields_test_suite); MODULE_AUTHOR("Johannes Berg <johannes@sipsolutions.net>"); MODULE_LICENSE("GPL"); diff --git a/lib/bitmap.c b/lib/bitmap.c index 4250519d7d1c..926408883456 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -3,17 +3,19 @@ * lib/bitmap.c * Helper functions for bitmap.h. */ -#include <linux/export.h> -#include <linux/thread_info.h> -#include <linux/ctype.h> -#include <linux/errno.h> + #include <linux/bitmap.h> #include <linux/bitops.h> #include <linux/bug.h> +#include <linux/ctype.h> +#include <linux/device.h> +#include <linux/errno.h> +#include <linux/export.h> #include <linux/kernel.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/string.h> +#include <linux/thread_info.h> #include <linux/uaccess.h> #include <asm/page.h> @@ -23,7 +25,7 @@ /** * DOC: bitmap introduction * - * bitmaps provide an array of bits, implemented using an an + * bitmaps provide an array of bits, implemented using an * array of unsigned longs. The number of valid bits in a * given bitmap does _not_ need to be an exact multiple of * BITS_PER_LONG. @@ -168,6 +170,73 @@ void __bitmap_shift_left(unsigned long *dst, const unsigned long *src, } EXPORT_SYMBOL(__bitmap_shift_left); +/** + * bitmap_cut() - remove bit region from bitmap and right shift remaining bits + * @dst: destination bitmap, might overlap with src + * @src: source bitmap + * @first: start bit of region to be removed + * @cut: number of bits to remove + * @nbits: bitmap size, in bits + * + * Set the n-th bit of @dst iff the n-th bit of @src is set and + * n is less than @first, or the m-th bit of @src is set for any + * m such that @first <= n < nbits, and m = n + @cut. + * + * In pictures, example for a big-endian 32-bit architecture: + * + * The @src bitmap is:: + * + * 31 63 + * | | + * 10000000 11000001 11110010 00010101 10000000 11000001 01110010 00010101 + * | | | | + * 16 14 0 32 + * + * if @cut is 3, and @first is 14, bits 14-16 in @src are cut and @dst is:: + * + * 31 63 + * | | + * 10110000 00011000 00110010 00010101 00010000 00011000 00101110 01000010 + * | | | + * 14 (bit 17 0 32 + * from @src) + * + * Note that @dst and @src might overlap partially or entirely. + * + * This is implemented in the obvious way, with a shift and carry + * step for each moved bit. Optimisation is left as an exercise + * for the compiler. + */ +void bitmap_cut(unsigned long *dst, const unsigned long *src, + unsigned int first, unsigned int cut, unsigned int nbits) +{ + unsigned int len = BITS_TO_LONGS(nbits); + unsigned long keep = 0, carry; + int i; + + if (first % BITS_PER_LONG) { + keep = src[first / BITS_PER_LONG] & + (~0UL >> (BITS_PER_LONG - first % BITS_PER_LONG)); + } + + memmove(dst, src, len * sizeof(*dst)); + + while (cut--) { + for (i = first / BITS_PER_LONG; i < len; i++) { + if (i < len - 1) + carry = dst[i + 1] & 1UL; + else + carry = 0; + + dst[i] = (dst[i] >> 1) | (carry << (BITS_PER_LONG - 1)); + } + } + + dst[first / BITS_PER_LONG] &= ~0UL << (first % BITS_PER_LONG); + dst[first / BITS_PER_LONG] |= keep; +} +EXPORT_SYMBOL(bitmap_cut); + int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { @@ -365,97 +434,6 @@ EXPORT_SYMBOL(bitmap_find_next_zero_area_off); * second version by Paul Jackson, third by Joe Korty. */ -#define CHUNKSZ 32 -#define nbits_to_hold_value(val) fls(val) -#define BASEDEC 10 /* fancier cpuset lists input in decimal */ - -/** - * __bitmap_parse - convert an ASCII hex string into a bitmap. - * @buf: pointer to buffer containing string. - * @buflen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0. - * @is_user: location of buffer, 0 indicates kernel space - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - * - * Commas group hex digits into chunks. Each chunk defines exactly 32 - * bits of the resultant bitmask. No chunk may specify a value larger - * than 32 bits (%-EOVERFLOW), and if a chunk specifies a smaller value - * then leading 0-bits are prepended. %-EINVAL is returned for illegal - * characters and for grouping errors such as "1,,5", ",44", "," and "". - * Leading and trailing whitespace accepted, but not embedded whitespace. - */ -int __bitmap_parse(const char *buf, unsigned int buflen, - int is_user, unsigned long *maskp, - int nmaskbits) -{ - int c, old_c, totaldigits, ndigits, nchunks, nbits; - u32 chunk; - const char __user __force *ubuf = (const char __user __force *)buf; - - bitmap_zero(maskp, nmaskbits); - - nchunks = nbits = totaldigits = c = 0; - do { - chunk = 0; - ndigits = totaldigits; - - /* Get the next chunk of the bitmap */ - while (buflen) { - old_c = c; - if (is_user) { - if (__get_user(c, ubuf++)) - return -EFAULT; - } - else - c = *buf++; - buflen--; - if (isspace(c)) - continue; - - /* - * If the last character was a space and the current - * character isn't '\0', we've got embedded whitespace. - * This is a no-no, so throw an error. - */ - if (totaldigits && c && isspace(old_c)) - return -EINVAL; - - /* A '\0' or a ',' signal the end of the chunk */ - if (c == '\0' || c == ',') - break; - - if (!isxdigit(c)) - return -EINVAL; - - /* - * Make sure there are at least 4 free bits in 'chunk'. - * If not, this hexdigit will overflow 'chunk', so - * throw an error. - */ - if (chunk & ~((1UL << (CHUNKSZ - 4)) - 1)) - return -EOVERFLOW; - - chunk = (chunk << 4) | hex_to_bin(c); - totaldigits++; - } - if (ndigits == totaldigits) - return -EINVAL; - if (nchunks == 0 && chunk == 0) - continue; - - __bitmap_shift_left(maskp, maskp, CHUNKSZ, nmaskbits); - *maskp |= chunk; - nchunks++; - nbits += (nchunks == 1) ? nbits_to_hold_value(chunk) : CHUNKSZ; - if (nbits > nmaskbits) - return -EOVERFLOW; - } while (buflen && c == ','); - - return 0; -} -EXPORT_SYMBOL(__bitmap_parse); - /** * bitmap_parse_user - convert an ASCII hex string in a user buffer into a bitmap * @@ -464,22 +442,22 @@ EXPORT_SYMBOL(__bitmap_parse); * then it must be terminated with a \0. * @maskp: pointer to bitmap array that will contain result. * @nmaskbits: size of bitmap, in bits. - * - * Wrapper for __bitmap_parse(), providing it with user buffer. - * - * We cannot have this as an inline function in bitmap.h because it needs - * linux/uaccess.h to get the access_ok() declaration and this causes - * cyclic dependencies. */ int bitmap_parse_user(const char __user *ubuf, unsigned int ulen, unsigned long *maskp, int nmaskbits) { - if (!access_ok(ubuf, ulen)) - return -EFAULT; - return __bitmap_parse((const char __force *)ubuf, - ulen, 1, maskp, nmaskbits); + char *buf; + int ret; + buf = memdup_user_nul(ubuf, ulen); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + ret = bitmap_parse(buf, UINT_MAX, maskp, nmaskbits); + + kfree(buf); + return ret; } EXPORT_SYMBOL(bitmap_parse_user); @@ -509,32 +487,148 @@ int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, } EXPORT_SYMBOL(bitmap_print_to_pagebuf); +/** + * bitmap_print_to_buf - convert bitmap to list or hex format ASCII string + * @list: indicates whether the bitmap must be list + * true: print in decimal list format + * false: print in hexadecimal bitmask format + */ +static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + const char *fmt = list ? "%*pbl\n" : "%*pb\n"; + ssize_t size; + void *data; + + data = kasprintf(GFP_KERNEL, fmt, nmaskbits, maskp); + if (!data) + return -ENOMEM; + + size = memory_read_from_buffer(buf, count, &off, data, strlen(data) + 1); + kfree(data); + + return size; +} + +/** + * bitmap_print_bitmask_to_buf - convert bitmap to hex bitmask format ASCII string + * + * The bitmap_print_to_pagebuf() is used indirectly via its cpumap wrapper + * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal + * bitmask and decimal list to userspace by sysfs ABI. + * Drivers might be using a normal attribute for this kind of ABIs. A + * normal attribute typically has show entry as below: + * static ssize_t example_attribute_show(struct device *dev, + * struct device_attribute *attr, char *buf) + * { + * ... + * return bitmap_print_to_pagebuf(true, buf, &mask, nr_trig_max); + * } + * show entry of attribute has no offset and count parameters and this + * means the file is limited to one page only. + * bitmap_print_to_pagebuf() API works terribly well for this kind of + * normal attribute with buf parameter and without offset, count: + * bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, + * int nmaskbits) + * { + * } + * The problem is once we have a large bitmap, we have a chance to get a + * bitmask or list more than one page. Especially for list, it could be + * as complex as 0,3,5,7,9,... We have no simple way to know it exact size. + * It turns out bin_attribute is a way to break this limit. bin_attribute + * has show entry as below: + * static ssize_t + * example_bin_attribute_show(struct file *filp, struct kobject *kobj, + * struct bin_attribute *attr, char *buf, + * loff_t offset, size_t count) + * { + * ... + * } + * With the new offset and count parameters, this makes sysfs ABI be able + * to support file size more than one page. For example, offset could be + * >= 4096. + * bitmap_print_bitmask_to_buf(), bitmap_print_list_to_buf() wit their + * cpumap wrapper cpumap_print_bitmask_to_buf(), cpumap_print_list_to_buf() + * make those drivers be able to support large bitmask and list after they + * move to use bin_attribute. In result, we have to pass the corresponding + * parameters such as off, count from bin_attribute show entry to this API. + * + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print + * + * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf() + * is similar with cpumap_print_to_pagebuf(), the difference is that + * bitmap_print_to_pagebuf() mainly serves sysfs attribute with the assumption + * the destination buffer is exactly one page and won't be more than one page. + * cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf(), on the other + * hand, mainly serves bin_attribute which doesn't work with exact one page, + * and it can break the size limit of converted decimal list and hexadecimal + * bitmask. + * + * WARNING! + * + * This function is not a replacement for sprintf() or bitmap_print_to_pagebuf(). + * It is intended to workaround sysfs limitations discussed above and should be + * used carefully in general case for the following reasons: + * - Time complexity is O(nbits^2/count), comparing to O(nbits) for snprintf(). + * - Memory complexity is O(nbits), comparing to O(1) for snprintf(). + * - @off and @count are NOT offset and number of bits to print. + * - If printing part of bitmap as list, the resulting string is not a correct + * list representation of bitmap. Particularly, some bits within or out of + * related interval may be erroneously set or unset. The format of the string + * may be broken, so bitmap_parselist-like parser may fail parsing it. + * - If printing the whole bitmap as list by parts, user must ensure the order + * of calls of the function such that the offset is incremented linearly. + * - If printing the whole bitmap as list by parts, user must keep bitmap + * unchanged between the very first and very last call. Otherwise concatenated + * result may be incorrect, and format may be broken. + * + * Returns the number of characters actually printed to @buf + */ +int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + return bitmap_print_to_buf(false, buf, maskp, nmaskbits, off, count); +} +EXPORT_SYMBOL(bitmap_print_bitmask_to_buf); + +/** + * bitmap_print_list_to_buf - convert bitmap to decimal list format ASCII string + * + * Everything is same with the above bitmap_print_bitmask_to_buf() except + * the print format. + */ +int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + return bitmap_print_to_buf(true, buf, maskp, nmaskbits, off, count); +} +EXPORT_SYMBOL(bitmap_print_list_to_buf); + /* * Region 9-38:4/10 describes the following bitmap structure: - * 0 9 12 18 38 - * .........****......****......****...... - * ^ ^ ^ ^ - * start off group_len end + * 0 9 12 18 38 N + * .........****......****......****.................. + * ^ ^ ^ ^ ^ + * start off group_len end nbits */ struct region { unsigned int start; unsigned int off; unsigned int group_len; unsigned int end; + unsigned int nbits; }; -static int bitmap_set_region(const struct region *r, - unsigned long *bitmap, int nbits) +static void bitmap_set_region(const struct region *r, unsigned long *bitmap) { unsigned int start; - if (r->end >= nbits) - return -ERANGE; - for (start = r->start; start <= r->end; start += r->group_len) bitmap_set(bitmap, start, min(r->end - start + 1, r->off)); - - return 0; } static int bitmap_check_region(const struct region *r) @@ -542,14 +636,23 @@ static int bitmap_check_region(const struct region *r) if (r->start > r->end || r->group_len == 0 || r->off > r->group_len) return -EINVAL; + if (r->end >= r->nbits) + return -ERANGE; + return 0; } -static const char *bitmap_getnum(const char *str, unsigned int *num) +static const char *bitmap_getnum(const char *str, unsigned int *num, + unsigned int lastbit) { unsigned long long n; unsigned int len; + if (str[0] == 'N') { + *num = lastbit; + return str + 1; + } + len = _parse_integer(str, 10, &n); if (!len) return ERR_PTR(-EINVAL); @@ -576,7 +679,7 @@ static inline bool end_of_region(char c) } /* - * The format allows commas and whitespases at the beginning + * The format allows commas and whitespaces at the beginning * of the region. */ static const char *bitmap_find_region(const char *str) @@ -587,9 +690,27 @@ static const char *bitmap_find_region(const char *str) return end_of_str(*str) ? NULL : str; } +static const char *bitmap_find_region_reverse(const char *start, const char *end) +{ + while (start <= end && __end_of_region(*end)) + end--; + + return end; +} + static const char *bitmap_parse_region(const char *str, struct region *r) { - str = bitmap_getnum(str, &r->start); + unsigned int lastbit = r->nbits - 1; + + if (!strncasecmp(str, "all", 3)) { + r->start = 0; + r->end = lastbit; + str += 3; + + goto check_pattern; + } + + str = bitmap_getnum(str, &r->start, lastbit); if (IS_ERR(str)) return str; @@ -599,24 +720,25 @@ static const char *bitmap_parse_region(const char *str, struct region *r) if (*str != '-') return ERR_PTR(-EINVAL); - str = bitmap_getnum(str + 1, &r->end); + str = bitmap_getnum(str + 1, &r->end, lastbit); if (IS_ERR(str)) return str; +check_pattern: if (end_of_region(*str)) goto no_pattern; if (*str != ':') return ERR_PTR(-EINVAL); - str = bitmap_getnum(str + 1, &r->off); + str = bitmap_getnum(str + 1, &r->off, lastbit); if (IS_ERR(str)) return str; if (*str != '/') return ERR_PTR(-EINVAL); - return bitmap_getnum(str + 1, &r->group_len); + return bitmap_getnum(str + 1, &r->group_len, lastbit); no_end: r->end = r->start; @@ -643,6 +765,10 @@ no_pattern: * From each group will be used only defined amount of bits. * Syntax: range:used_size/group_size * Example: 0-1023:2/256 ==> 0,1,256,257,512,513,768,769 + * The value 'N' can be used as a dynamically substituted token for the + * maximum allowed value; i.e (nmaskbits - 1). Keep in mind that it is + * dynamic, so if system changes cause the bitmap width to change, such + * as more cores in a CPU list, then any ranges using N will also change. * * Returns: 0 on success, -errno on invalid input strings. Error values: * @@ -656,7 +782,8 @@ int bitmap_parselist(const char *buf, unsigned long *maskp, int nmaskbits) struct region r; long ret; - bitmap_zero(maskp, nmaskbits); + r.nbits = nmaskbits; + bitmap_zero(maskp, r.nbits); while (buf) { buf = bitmap_find_region(buf); @@ -671,9 +798,7 @@ int bitmap_parselist(const char *buf, unsigned long *maskp, int nmaskbits) if (ret) return ret; - ret = bitmap_set_region(&r, maskp, nmaskbits); - if (ret) - return ret; + bitmap_set_region(&r, maskp); } return 0; @@ -710,8 +835,85 @@ int bitmap_parselist_user(const char __user *ubuf, } EXPORT_SYMBOL(bitmap_parselist_user); +static const char *bitmap_get_x32_reverse(const char *start, + const char *end, u32 *num) +{ + u32 ret = 0; + int c, i; + + for (i = 0; i < 32; i += 4) { + c = hex_to_bin(*end--); + if (c < 0) + return ERR_PTR(-EINVAL); + + ret |= c << i; + + if (start > end || __end_of_region(*end)) + goto out; + } + + if (hex_to_bin(*end--) >= 0) + return ERR_PTR(-EOVERFLOW); +out: + *num = ret; + return end; +} + +/** + * bitmap_parse - convert an ASCII hex string into a bitmap. + * @start: pointer to buffer containing string. + * @buflen: buffer size in bytes. If string is smaller than this + * then it must be terminated with a \0 or \n. In that case, + * UINT_MAX may be provided instead of string length. + * @maskp: pointer to bitmap array that will contain result. + * @nmaskbits: size of bitmap, in bits. + * + * Commas group hex digits into chunks. Each chunk defines exactly 32 + * bits of the resultant bitmask. No chunk may specify a value larger + * than 32 bits (%-EOVERFLOW), and if a chunk specifies a smaller value + * then leading 0-bits are prepended. %-EINVAL is returned for illegal + * characters. Grouping such as "1,,5", ",44", "," or "" is allowed. + * Leading, embedded and trailing whitespace accepted. + */ +int bitmap_parse(const char *start, unsigned int buflen, + unsigned long *maskp, int nmaskbits) +{ + const char *end = strnchrnul(start, buflen, '\n') - 1; + int chunks = BITS_TO_U32(nmaskbits); + u32 *bitmap = (u32 *)maskp; + int unset_bit; + int chunk; + + for (chunk = 0; ; chunk++) { + end = bitmap_find_region_reverse(start, end); + if (start > end) + break; + + if (!chunks--) + return -EOVERFLOW; + +#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN) + end = bitmap_get_x32_reverse(start, end, &bitmap[chunk ^ 1]); +#else + end = bitmap_get_x32_reverse(start, end, &bitmap[chunk]); +#endif + if (IS_ERR(end)) + return PTR_ERR(end); + } + + unset_bit = (BITS_TO_U32(nmaskbits) - chunks) * 32; + if (unset_bit < nmaskbits) { + bitmap_clear(maskp, unset_bit, nmaskbits - unset_bit); + return 0; + } + + if (find_next_bit(maskp, unset_bit, nmaskbits) != unset_bit) + return -EOVERFLOW; + + return 0; +} +EXPORT_SYMBOL(bitmap_parse); -#ifdef CONFIG_NUMA /** * bitmap_pos_to_ord - find ordinal of set bit at given position in bitmap * @buf: pointer to a bitmap @@ -820,6 +1022,7 @@ void bitmap_remap(unsigned long *dst, const unsigned long *src, set_bit(bitmap_ord_to_pos(new, n % w, nbits), dst); } } +EXPORT_SYMBOL(bitmap_remap); /** * bitmap_bitremap - Apply map defined by a pair of bitmaps to a single bit @@ -857,7 +1060,9 @@ int bitmap_bitremap(int oldbit, const unsigned long *old, else return bitmap_ord_to_pos(new, n % w, bits); } +EXPORT_SYMBOL(bitmap_bitremap); +#ifdef CONFIG_NUMA /** * bitmap_onto - translate one bitmap relative to another * @dst: resulting translated bitmap @@ -1193,12 +1398,57 @@ unsigned long *bitmap_zalloc(unsigned int nbits, gfp_t flags) } EXPORT_SYMBOL(bitmap_zalloc); +unsigned long *bitmap_alloc_node(unsigned int nbits, gfp_t flags, int node) +{ + return kmalloc_array_node(BITS_TO_LONGS(nbits), sizeof(unsigned long), + flags, node); +} +EXPORT_SYMBOL(bitmap_alloc_node); + +unsigned long *bitmap_zalloc_node(unsigned int nbits, gfp_t flags, int node) +{ + return bitmap_alloc_node(nbits, flags | __GFP_ZERO, node); +} +EXPORT_SYMBOL(bitmap_zalloc_node); + void bitmap_free(const unsigned long *bitmap) { kfree(bitmap); } EXPORT_SYMBOL(bitmap_free); +static void devm_bitmap_free(void *data) +{ + unsigned long *bitmap = data; + + bitmap_free(bitmap); +} + +unsigned long *devm_bitmap_alloc(struct device *dev, + unsigned int nbits, gfp_t flags) +{ + unsigned long *bitmap; + int ret; + + bitmap = bitmap_alloc(nbits, flags); + if (!bitmap) + return NULL; + + ret = devm_add_action_or_reset(dev, devm_bitmap_free, bitmap); + if (ret) + return NULL; + + return bitmap; +} +EXPORT_SYMBOL_GPL(devm_bitmap_alloc); + +unsigned long *devm_bitmap_zalloc(struct device *dev, + unsigned int nbits, gfp_t flags) +{ + return devm_bitmap_alloc(dev, nbits, flags | __GFP_ZERO); +} +EXPORT_SYMBOL_GPL(devm_bitmap_zalloc); + #if BITS_PER_LONG == 64 /** * bitmap_from_arr32 - copy the contents of u32 array of bits to bitmap diff --git a/lib/bootconfig.c b/lib/bootconfig.c new file mode 100644 index 000000000000..74f3201ab8e5 --- /dev/null +++ b/lib/bootconfig.c @@ -0,0 +1,972 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Extra Boot Config + * Masami Hiramatsu <mhiramat@kernel.org> + */ + +#ifdef __KERNEL__ +#include <linux/bootconfig.h> +#include <linux/bug.h> +#include <linux/ctype.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/memblock.h> +#include <linux/string.h> +#else /* !__KERNEL__ */ +/* + * NOTE: This is only for tools/bootconfig, because tools/bootconfig will + * run the parser sanity test. + * This does NOT mean lib/bootconfig.c is available in the user space. + * However, if you change this file, please make sure the tools/bootconfig + * has no issue on building and running. + */ +#include <linux/bootconfig.h> +#endif + +/* + * Extra Boot Config (XBC) is given as tree-structured ascii text of + * key-value pairs on memory. + * xbc_parse() parses the text to build a simple tree. Each tree node is + * simply a key word or a value. A key node may have a next key node or/and + * a child node (both key and value). A value node may have a next value + * node (for array). + */ + +static struct xbc_node *xbc_nodes __initdata; +static int xbc_node_num __initdata; +static char *xbc_data __initdata; +static size_t xbc_data_size __initdata; +static struct xbc_node *last_parent __initdata; +static const char *xbc_err_msg __initdata; +static int xbc_err_pos __initdata; +static int open_brace[XBC_DEPTH_MAX] __initdata; +static int brace_index __initdata; + +#ifdef __KERNEL__ +static inline void * __init xbc_alloc_mem(size_t size) +{ + return memblock_alloc(size, SMP_CACHE_BYTES); +} + +static inline void __init xbc_free_mem(void *addr, size_t size) +{ + memblock_free(addr, size); +} + +#else /* !__KERNEL__ */ + +static inline void *xbc_alloc_mem(size_t size) +{ + return malloc(size); +} + +static inline void xbc_free_mem(void *addr, size_t size) +{ + free(addr); +} +#endif +/** + * xbc_get_info() - Get the information of loaded boot config + * @node_size: A pointer to store the number of nodes. + * @data_size: A pointer to store the size of bootconfig data. + * + * Get the number of used nodes in @node_size if it is not NULL, + * and the size of bootconfig data in @data_size if it is not NULL. + * Return 0 if the boot config is initialized, or return -ENODEV. + */ +int __init xbc_get_info(int *node_size, size_t *data_size) +{ + if (!xbc_data) + return -ENODEV; + + if (node_size) + *node_size = xbc_node_num; + if (data_size) + *data_size = xbc_data_size; + return 0; +} + +static int __init xbc_parse_error(const char *msg, const char *p) +{ + xbc_err_msg = msg; + xbc_err_pos = (int)(p - xbc_data); + + return -EINVAL; +} + +/** + * xbc_root_node() - Get the root node of extended boot config + * + * Return the address of root node of extended boot config. If the + * extended boot config is not initiized, return NULL. + */ +struct xbc_node * __init xbc_root_node(void) +{ + if (unlikely(!xbc_data)) + return NULL; + + return xbc_nodes; +} + +/** + * xbc_node_index() - Get the index of XBC node + * @node: A target node of getting index. + * + * Return the index number of @node in XBC node list. + */ +int __init xbc_node_index(struct xbc_node *node) +{ + return node - &xbc_nodes[0]; +} + +/** + * xbc_node_get_parent() - Get the parent XBC node + * @node: An XBC node. + * + * Return the parent node of @node. If the node is top node of the tree, + * return NULL. + */ +struct xbc_node * __init xbc_node_get_parent(struct xbc_node *node) +{ + return node->parent == XBC_NODE_MAX ? NULL : &xbc_nodes[node->parent]; +} + +/** + * xbc_node_get_child() - Get the child XBC node + * @node: An XBC node. + * + * Return the first child node of @node. If the node has no child, return + * NULL. + */ +struct xbc_node * __init xbc_node_get_child(struct xbc_node *node) +{ + return node->child ? &xbc_nodes[node->child] : NULL; +} + +/** + * xbc_node_get_next() - Get the next sibling XBC node + * @node: An XBC node. + * + * Return the NEXT sibling node of @node. If the node has no next sibling, + * return NULL. Note that even if this returns NULL, it doesn't mean @node + * has no siblings. (You also has to check whether the parent's child node + * is @node or not.) + */ +struct xbc_node * __init xbc_node_get_next(struct xbc_node *node) +{ + return node->next ? &xbc_nodes[node->next] : NULL; +} + +/** + * xbc_node_get_data() - Get the data of XBC node + * @node: An XBC node. + * + * Return the data (which is always a null terminated string) of @node. + * If the node has invalid data, warn and return NULL. + */ +const char * __init xbc_node_get_data(struct xbc_node *node) +{ + int offset = node->data & ~XBC_VALUE; + + if (WARN_ON(offset >= xbc_data_size)) + return NULL; + + return xbc_data + offset; +} + +static bool __init +xbc_node_match_prefix(struct xbc_node *node, const char **prefix) +{ + const char *p = xbc_node_get_data(node); + int len = strlen(p); + + if (strncmp(*prefix, p, len)) + return false; + + p = *prefix + len; + if (*p == '.') + p++; + else if (*p != '\0') + return false; + *prefix = p; + + return true; +} + +/** + * xbc_node_find_subkey() - Find a subkey node which matches given key + * @parent: An XBC node. + * @key: A key string. + * + * Search a key node under @parent which matches @key. The @key can contain + * several words jointed with '.'. If @parent is NULL, this searches the + * node from whole tree. Return NULL if no node is matched. + */ +struct xbc_node * __init +xbc_node_find_subkey(struct xbc_node *parent, const char *key) +{ + struct xbc_node *node; + + if (parent) + node = xbc_node_get_subkey(parent); + else + node = xbc_root_node(); + + while (node && xbc_node_is_key(node)) { + if (!xbc_node_match_prefix(node, &key)) + node = xbc_node_get_next(node); + else if (*key != '\0') + node = xbc_node_get_subkey(node); + else + break; + } + + return node; +} + +/** + * xbc_node_find_value() - Find a value node which matches given key + * @parent: An XBC node. + * @key: A key string. + * @vnode: A container pointer of found XBC node. + * + * Search a value node under @parent whose (parent) key node matches @key, + * store it in *@vnode, and returns the value string. + * The @key can contain several words jointed with '.'. If @parent is NULL, + * this searches the node from whole tree. Return the value string if a + * matched key found, return NULL if no node is matched. + * Note that this returns 0-length string and stores NULL in *@vnode if the + * key has no value. And also it will return the value of the first entry if + * the value is an array. + */ +const char * __init +xbc_node_find_value(struct xbc_node *parent, const char *key, + struct xbc_node **vnode) +{ + struct xbc_node *node = xbc_node_find_subkey(parent, key); + + if (!node || !xbc_node_is_key(node)) + return NULL; + + node = xbc_node_get_child(node); + if (node && !xbc_node_is_value(node)) + return NULL; + + if (vnode) + *vnode = node; + + return node ? xbc_node_get_data(node) : ""; +} + +/** + * xbc_node_compose_key_after() - Compose partial key string of the XBC node + * @root: Root XBC node + * @node: Target XBC node. + * @buf: A buffer to store the key. + * @size: The size of the @buf. + * + * Compose the partial key of the @node into @buf, which is starting right + * after @root (@root is not included.) If @root is NULL, this returns full + * key words of @node. + * Returns the total length of the key stored in @buf. Returns -EINVAL + * if @node is NULL or @root is not the ancestor of @node or @root is @node, + * or returns -ERANGE if the key depth is deeper than max depth. + * This is expected to be used with xbc_find_node() to list up all (child) + * keys under given key. + */ +int __init xbc_node_compose_key_after(struct xbc_node *root, + struct xbc_node *node, + char *buf, size_t size) +{ + uint16_t keys[XBC_DEPTH_MAX]; + int depth = 0, ret = 0, total = 0; + + if (!node || node == root) + return -EINVAL; + + if (xbc_node_is_value(node)) + node = xbc_node_get_parent(node); + + while (node && node != root) { + keys[depth++] = xbc_node_index(node); + if (depth == XBC_DEPTH_MAX) + return -ERANGE; + node = xbc_node_get_parent(node); + } + if (!node && root) + return -EINVAL; + + while (--depth >= 0) { + node = xbc_nodes + keys[depth]; + ret = snprintf(buf, size, "%s%s", xbc_node_get_data(node), + depth ? "." : ""); + if (ret < 0) + return ret; + if (ret > size) { + size = 0; + } else { + size -= ret; + buf += ret; + } + total += ret; + } + + return total; +} + +/** + * xbc_node_find_next_leaf() - Find the next leaf node under given node + * @root: An XBC root node + * @node: An XBC node which starts from. + * + * Search the next leaf node (which means the terminal key node) of @node + * under @root node (including @root node itself). + * Return the next node or NULL if next leaf node is not found. + */ +struct xbc_node * __init xbc_node_find_next_leaf(struct xbc_node *root, + struct xbc_node *node) +{ + struct xbc_node *next; + + if (unlikely(!xbc_data)) + return NULL; + + if (!node) { /* First try */ + node = root; + if (!node) + node = xbc_nodes; + } else { + /* Leaf node may have a subkey */ + next = xbc_node_get_subkey(node); + if (next) { + node = next; + goto found; + } + + if (node == root) /* @root was a leaf, no child node. */ + return NULL; + + while (!node->next) { + node = xbc_node_get_parent(node); + if (node == root) + return NULL; + /* User passed a node which is not uder parent */ + if (WARN_ON(!node)) + return NULL; + } + node = xbc_node_get_next(node); + } + +found: + while (node && !xbc_node_is_leaf(node)) + node = xbc_node_get_child(node); + + return node; +} + +/** + * xbc_node_find_next_key_value() - Find the next key-value pair nodes + * @root: An XBC root node + * @leaf: A container pointer of XBC node which starts from. + * + * Search the next leaf node (which means the terminal key node) of *@leaf + * under @root node. Returns the value and update *@leaf if next leaf node + * is found, or NULL if no next leaf node is found. + * Note that this returns 0-length string if the key has no value, or + * the value of the first entry if the value is an array. + */ +const char * __init xbc_node_find_next_key_value(struct xbc_node *root, + struct xbc_node **leaf) +{ + /* tip must be passed */ + if (WARN_ON(!leaf)) + return NULL; + + *leaf = xbc_node_find_next_leaf(root, *leaf); + if (!*leaf) + return NULL; + if ((*leaf)->child) + return xbc_node_get_data(xbc_node_get_child(*leaf)); + else + return ""; /* No value key */ +} + +/* XBC parse and tree build */ + +static int __init xbc_init_node(struct xbc_node *node, char *data, uint32_t flag) +{ + unsigned long offset = data - xbc_data; + + if (WARN_ON(offset >= XBC_DATA_MAX)) + return -EINVAL; + + node->data = (uint16_t)offset | flag; + node->child = 0; + node->next = 0; + + return 0; +} + +static struct xbc_node * __init xbc_add_node(char *data, uint32_t flag) +{ + struct xbc_node *node; + + if (xbc_node_num == XBC_NODE_MAX) + return NULL; + + node = &xbc_nodes[xbc_node_num++]; + if (xbc_init_node(node, data, flag) < 0) + return NULL; + + return node; +} + +static inline __init struct xbc_node *xbc_last_sibling(struct xbc_node *node) +{ + while (node->next) + node = xbc_node_get_next(node); + + return node; +} + +static inline __init struct xbc_node *xbc_last_child(struct xbc_node *node) +{ + while (node->child) + node = xbc_node_get_child(node); + + return node; +} + +static struct xbc_node * __init __xbc_add_sibling(char *data, uint32_t flag, bool head) +{ + struct xbc_node *sib, *node = xbc_add_node(data, flag); + + if (node) { + if (!last_parent) { + /* Ignore @head in this case */ + node->parent = XBC_NODE_MAX; + sib = xbc_last_sibling(xbc_nodes); + sib->next = xbc_node_index(node); + } else { + node->parent = xbc_node_index(last_parent); + if (!last_parent->child || head) { + node->next = last_parent->child; + last_parent->child = xbc_node_index(node); + } else { + sib = xbc_node_get_child(last_parent); + sib = xbc_last_sibling(sib); + sib->next = xbc_node_index(node); + } + } + } else + xbc_parse_error("Too many nodes", data); + + return node; +} + +static inline struct xbc_node * __init xbc_add_sibling(char *data, uint32_t flag) +{ + return __xbc_add_sibling(data, flag, false); +} + +static inline struct xbc_node * __init xbc_add_head_sibling(char *data, uint32_t flag) +{ + return __xbc_add_sibling(data, flag, true); +} + +static inline __init struct xbc_node *xbc_add_child(char *data, uint32_t flag) +{ + struct xbc_node *node = xbc_add_sibling(data, flag); + + if (node) + last_parent = node; + + return node; +} + +static inline __init bool xbc_valid_keyword(char *key) +{ + if (key[0] == '\0') + return false; + + while (isalnum(*key) || *key == '-' || *key == '_') + key++; + + return *key == '\0'; +} + +static char *skip_comment(char *p) +{ + char *ret; + + ret = strchr(p, '\n'); + if (!ret) + ret = p + strlen(p); + else + ret++; + + return ret; +} + +static char *skip_spaces_until_newline(char *p) +{ + while (isspace(*p) && *p != '\n') + p++; + return p; +} + +static int __init __xbc_open_brace(char *p) +{ + /* Push the last key as open brace */ + open_brace[brace_index++] = xbc_node_index(last_parent); + if (brace_index >= XBC_DEPTH_MAX) + return xbc_parse_error("Exceed max depth of braces", p); + + return 0; +} + +static int __init __xbc_close_brace(char *p) +{ + brace_index--; + if (!last_parent || brace_index < 0 || + (open_brace[brace_index] != xbc_node_index(last_parent))) + return xbc_parse_error("Unexpected closing brace", p); + + if (brace_index == 0) + last_parent = NULL; + else + last_parent = &xbc_nodes[open_brace[brace_index - 1]]; + + return 0; +} + +/* + * Return delimiter or error, no node added. As same as lib/cmdline.c, + * you can use " around spaces, but can't escape " for value. + */ +static int __init __xbc_parse_value(char **__v, char **__n) +{ + char *p, *v = *__v; + int c, quotes = 0; + + v = skip_spaces(v); + while (*v == '#') { + v = skip_comment(v); + v = skip_spaces(v); + } + if (*v == '"' || *v == '\'') { + quotes = *v; + v++; + } + p = v - 1; + while ((c = *++p)) { + if (!isprint(c) && !isspace(c)) + return xbc_parse_error("Non printable value", p); + if (quotes) { + if (c != quotes) + continue; + quotes = 0; + *p++ = '\0'; + p = skip_spaces_until_newline(p); + c = *p; + if (c && !strchr(",;\n#}", c)) + return xbc_parse_error("No value delimiter", p); + if (*p) + p++; + break; + } + if (strchr(",;\n#}", c)) { + *p++ = '\0'; + v = strim(v); + break; + } + } + if (quotes) + return xbc_parse_error("No closing quotes", p); + if (c == '#') { + p = skip_comment(p); + c = '\n'; /* A comment must be treated as a newline */ + } + *__n = p; + *__v = v; + + return c; +} + +static int __init xbc_parse_array(char **__v) +{ + struct xbc_node *node; + char *next; + int c = 0; + + if (last_parent->child) + last_parent = xbc_node_get_child(last_parent); + + do { + c = __xbc_parse_value(__v, &next); + if (c < 0) + return c; + + node = xbc_add_child(*__v, XBC_VALUE); + if (!node) + return -ENOMEM; + *__v = next; + } while (c == ','); + node->child = 0; + + return c; +} + +static inline __init +struct xbc_node *find_match_node(struct xbc_node *node, char *k) +{ + while (node) { + if (!strcmp(xbc_node_get_data(node), k)) + break; + node = xbc_node_get_next(node); + } + return node; +} + +static int __init __xbc_add_key(char *k) +{ + struct xbc_node *node, *child; + + if (!xbc_valid_keyword(k)) + return xbc_parse_error("Invalid keyword", k); + + if (unlikely(xbc_node_num == 0)) + goto add_node; + + if (!last_parent) /* the first level */ + node = find_match_node(xbc_nodes, k); + else { + child = xbc_node_get_child(last_parent); + /* Since the value node is the first child, skip it. */ + if (child && xbc_node_is_value(child)) + child = xbc_node_get_next(child); + node = find_match_node(child, k); + } + + if (node) + last_parent = node; + else { +add_node: + node = xbc_add_child(k, XBC_KEY); + if (!node) + return -ENOMEM; + } + return 0; +} + +static int __init __xbc_parse_keys(char *k) +{ + char *p; + int ret; + + k = strim(k); + while ((p = strchr(k, '.'))) { + *p++ = '\0'; + ret = __xbc_add_key(k); + if (ret) + return ret; + k = p; + } + + return __xbc_add_key(k); +} + +static int __init xbc_parse_kv(char **k, char *v, int op) +{ + struct xbc_node *prev_parent = last_parent; + struct xbc_node *child; + char *next; + int c, ret; + + ret = __xbc_parse_keys(*k); + if (ret) + return ret; + + c = __xbc_parse_value(&v, &next); + if (c < 0) + return c; + + child = xbc_node_get_child(last_parent); + if (child && xbc_node_is_value(child)) { + if (op == '=') + return xbc_parse_error("Value is redefined", v); + if (op == ':') { + unsigned short nidx = child->next; + + xbc_init_node(child, v, XBC_VALUE); + child->next = nidx; /* keep subkeys */ + goto array; + } + /* op must be '+' */ + last_parent = xbc_last_child(child); + } + /* The value node should always be the first child */ + if (!xbc_add_head_sibling(v, XBC_VALUE)) + return -ENOMEM; + +array: + if (c == ',') { /* Array */ + c = xbc_parse_array(&next); + if (c < 0) + return c; + } + + last_parent = prev_parent; + + if (c == '}') { + ret = __xbc_close_brace(next - 1); + if (ret < 0) + return ret; + } + + *k = next; + + return 0; +} + +static int __init xbc_parse_key(char **k, char *n) +{ + struct xbc_node *prev_parent = last_parent; + int ret; + + *k = strim(*k); + if (**k != '\0') { + ret = __xbc_parse_keys(*k); + if (ret) + return ret; + last_parent = prev_parent; + } + *k = n; + + return 0; +} + +static int __init xbc_open_brace(char **k, char *n) +{ + int ret; + + ret = __xbc_parse_keys(*k); + if (ret) + return ret; + *k = n; + + return __xbc_open_brace(n - 1); +} + +static int __init xbc_close_brace(char **k, char *n) +{ + int ret; + + ret = xbc_parse_key(k, n); + if (ret) + return ret; + /* k is updated in xbc_parse_key() */ + + return __xbc_close_brace(n - 1); +} + +static int __init xbc_verify_tree(void) +{ + int i, depth, len, wlen; + struct xbc_node *n, *m; + + /* Brace closing */ + if (brace_index) { + n = &xbc_nodes[open_brace[brace_index]]; + return xbc_parse_error("Brace is not closed", + xbc_node_get_data(n)); + } + + /* Empty tree */ + if (xbc_node_num == 0) { + xbc_parse_error("Empty config", xbc_data); + return -ENOENT; + } + + for (i = 0; i < xbc_node_num; i++) { + if (xbc_nodes[i].next > xbc_node_num) { + return xbc_parse_error("No closing brace", + xbc_node_get_data(xbc_nodes + i)); + } + } + + /* Key tree limitation check */ + n = &xbc_nodes[0]; + depth = 1; + len = 0; + + while (n) { + wlen = strlen(xbc_node_get_data(n)) + 1; + len += wlen; + if (len > XBC_KEYLEN_MAX) + return xbc_parse_error("Too long key length", + xbc_node_get_data(n)); + + m = xbc_node_get_child(n); + if (m && xbc_node_is_key(m)) { + n = m; + depth++; + if (depth > XBC_DEPTH_MAX) + return xbc_parse_error("Too many key words", + xbc_node_get_data(n)); + continue; + } + len -= wlen; + m = xbc_node_get_next(n); + while (!m) { + n = xbc_node_get_parent(n); + if (!n) + break; + len -= strlen(xbc_node_get_data(n)) + 1; + depth--; + m = xbc_node_get_next(n); + } + n = m; + } + + return 0; +} + +/* Need to setup xbc_data and xbc_nodes before call this. */ +static int __init xbc_parse_tree(void) +{ + char *p, *q; + int ret = 0, c; + + last_parent = NULL; + p = xbc_data; + do { + q = strpbrk(p, "{}=+;:\n#"); + if (!q) { + p = skip_spaces(p); + if (*p != '\0') + ret = xbc_parse_error("No delimiter", p); + break; + } + + c = *q; + *q++ = '\0'; + switch (c) { + case ':': + case '+': + if (*q++ != '=') { + ret = xbc_parse_error(c == '+' ? + "Wrong '+' operator" : + "Wrong ':' operator", + q - 2); + break; + } + fallthrough; + case '=': + ret = xbc_parse_kv(&p, q, c); + break; + case '{': + ret = xbc_open_brace(&p, q); + break; + case '#': + q = skip_comment(q); + fallthrough; + case ';': + case '\n': + ret = xbc_parse_key(&p, q); + break; + case '}': + ret = xbc_close_brace(&p, q); + break; + } + } while (!ret); + + return ret; +} + +/** + * xbc_exit() - Clean up all parsed bootconfig + * + * This clears all data structures of parsed bootconfig on memory. + * If you need to reuse xbc_init() with new boot config, you can + * use this. + */ +void __init xbc_exit(void) +{ + xbc_free_mem(xbc_data, xbc_data_size); + xbc_data = NULL; + xbc_data_size = 0; + xbc_node_num = 0; + xbc_free_mem(xbc_nodes, sizeof(struct xbc_node) * XBC_NODE_MAX); + xbc_nodes = NULL; + brace_index = 0; +} + +/** + * xbc_init() - Parse given XBC file and build XBC internal tree + * @data: The boot config text original data + * @size: The size of @data + * @emsg: A pointer of const char * to store the error message + * @epos: A pointer of int to store the error position + * + * This parses the boot config text in @data. @size must be smaller + * than XBC_DATA_MAX. + * Return the number of stored nodes (>0) if succeeded, or -errno + * if there is any error. + * In error cases, @emsg will be updated with an error message and + * @epos will be updated with the error position which is the byte offset + * of @buf. If the error is not a parser error, @epos will be -1. + */ +int __init xbc_init(const char *data, size_t size, const char **emsg, int *epos) +{ + int ret; + + if (epos) + *epos = -1; + + if (xbc_data) { + if (emsg) + *emsg = "Bootconfig is already initialized"; + return -EBUSY; + } + if (size > XBC_DATA_MAX || size == 0) { + if (emsg) + *emsg = size ? "Config data is too big" : + "Config data is empty"; + return -ERANGE; + } + + xbc_data = xbc_alloc_mem(size + 1); + if (!xbc_data) { + if (emsg) + *emsg = "Failed to allocate bootconfig data"; + return -ENOMEM; + } + memcpy(xbc_data, data, size); + xbc_data[size] = '\0'; + xbc_data_size = size + 1; + + xbc_nodes = xbc_alloc_mem(sizeof(struct xbc_node) * XBC_NODE_MAX); + if (!xbc_nodes) { + if (emsg) + *emsg = "Failed to allocate bootconfig nodes"; + xbc_exit(); + return -ENOMEM; + } + memset(xbc_nodes, 0, sizeof(struct xbc_node) * XBC_NODE_MAX); + + ret = xbc_parse_tree(); + if (!ret) + ret = xbc_verify_tree(); + + if (ret < 0) { + if (epos) + *epos = xbc_err_pos; + if (emsg) + *emsg = xbc_err_msg; + xbc_exit(); + } else + ret = xbc_node_num; + + return ret; +} diff --git a/lib/bsearch.c b/lib/bsearch.c index 8b3aae5ae77a..bf86aa66f2b2 100644 --- a/lib/bsearch.c +++ b/lib/bsearch.c @@ -28,27 +28,9 @@ * the key and elements in the array are of the same type, you can use * the same comparison function for both sort() and bsearch(). */ -void *bsearch(const void *key, const void *base, size_t num, size_t size, - cmp_func_t cmp) +void *bsearch(const void *key, const void *base, size_t num, size_t size, cmp_func_t cmp) { - const char *pivot; - int result; - - while (num > 0) { - pivot = base + (num >> 1) * size; - result = cmp(key, pivot); - - if (result == 0) - return (void *)pivot; - - if (result > 0) { - base = pivot + size; - num--; - } - num >>= 1; - } - - return NULL; + return __inline_bsearch(key, base, num, size, cmp); } EXPORT_SYMBOL(bsearch); NOKPROBE_SYMBOL(bsearch); diff --git a/lib/bug.c b/lib/bug.c index 8c98af0bf585..45a0584f6541 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -47,6 +47,7 @@ #include <linux/bug.h> #include <linux/sched.h> #include <linux/rculist.h> +#include <linux/ftrace.h> extern struct bug_entry __start___bug_table[], __stop___bug_table[]; @@ -90,8 +91,6 @@ void module_bug_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, char *secstrings; unsigned int i; - lockdep_assert_held(&module_mutex); - mod->bug_table = NULL; mod->num_bugs = 0; @@ -117,7 +116,6 @@ void module_bug_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, void module_bug_cleanup(struct module *mod) { - lockdep_assert_held(&module_mutex); list_del_rcu(&mod->bug_list); } @@ -129,6 +127,22 @@ static inline struct bug_entry *module_find_bug(unsigned long bugaddr) } #endif +void bug_get_file_line(struct bug_entry *bug, const char **file, + unsigned int *line) +{ +#ifdef CONFIG_DEBUG_BUGVERBOSE +#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS + *file = bug->file; +#else + *file = (const char *)bug + bug->file_disp; +#endif + *line = bug->line; +#else + *file = NULL; + *line = 0; +#endif +} + struct bug_entry *find_bug(unsigned long bugaddr) { struct bug_entry *bug; @@ -153,32 +167,22 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) if (!bug) return BUG_TRAP_TYPE_NONE; - file = NULL; - line = 0; - warning = 0; + disable_trace_on_warning(); - if (bug) { -#ifdef CONFIG_DEBUG_BUGVERBOSE -#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS - file = bug->file; -#else - file = (const char *)bug + bug->file_disp; -#endif - line = bug->line; -#endif - warning = (bug->flags & BUGFLAG_WARNING) != 0; - once = (bug->flags & BUGFLAG_ONCE) != 0; - done = (bug->flags & BUGFLAG_DONE) != 0; - - if (warning && once) { - if (done) - return BUG_TRAP_TYPE_WARN; - - /* - * Since this is the only store, concurrency is not an issue. - */ - bug->flags |= BUGFLAG_DONE; - } + bug_get_file_line(bug, &file, &line); + + warning = (bug->flags & BUGFLAG_WARNING) != 0; + once = (bug->flags & BUGFLAG_ONCE) != 0; + done = (bug->flags & BUGFLAG_DONE) != 0; + + if (warning && once) { + if (done) + return BUG_TRAP_TYPE_WARN; + + /* + * Since this is the only store, concurrency is not an issue. + */ + bug->flags |= BUGFLAG_DONE; } /* diff --git a/lib/buildid.c b/lib/buildid.c new file mode 100644 index 000000000000..dfc62625cae4 --- /dev/null +++ b/lib/buildid.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/buildid.h> +#include <linux/cache.h> +#include <linux/elf.h> +#include <linux/kernel.h> +#include <linux/pagemap.h> + +#define BUILD_ID 3 + +/* + * Parse build id from the note segment. This logic can be shared between + * 32-bit and 64-bit system, because Elf32_Nhdr and Elf64_Nhdr are + * identical. + */ +static int parse_build_id_buf(unsigned char *build_id, + __u32 *size, + const void *note_start, + Elf32_Word note_size) +{ + Elf32_Word note_offs = 0, new_offs; + + while (note_offs + sizeof(Elf32_Nhdr) < note_size) { + Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs); + + if (nhdr->n_type == BUILD_ID && + nhdr->n_namesz == sizeof("GNU") && + !strcmp((char *)(nhdr + 1), "GNU") && + nhdr->n_descsz > 0 && + nhdr->n_descsz <= BUILD_ID_SIZE_MAX) { + memcpy(build_id, + note_start + note_offs + + ALIGN(sizeof("GNU"), 4) + sizeof(Elf32_Nhdr), + nhdr->n_descsz); + memset(build_id + nhdr->n_descsz, 0, + BUILD_ID_SIZE_MAX - nhdr->n_descsz); + if (size) + *size = nhdr->n_descsz; + return 0; + } + new_offs = note_offs + sizeof(Elf32_Nhdr) + + ALIGN(nhdr->n_namesz, 4) + ALIGN(nhdr->n_descsz, 4); + if (new_offs <= note_offs) /* overflow */ + break; + note_offs = new_offs; + } + + return -EINVAL; +} + +static inline int parse_build_id(const void *page_addr, + unsigned char *build_id, + __u32 *size, + const void *note_start, + Elf32_Word note_size) +{ + /* check for overflow */ + if (note_start < page_addr || note_start + note_size < note_start) + return -EINVAL; + + /* only supports note that fits in the first page */ + if (note_start + note_size > page_addr + PAGE_SIZE) + return -EINVAL; + + return parse_build_id_buf(build_id, size, note_start, note_size); +} + +/* Parse build ID from 32-bit ELF */ +static int get_build_id_32(const void *page_addr, unsigned char *build_id, + __u32 *size) +{ + Elf32_Ehdr *ehdr = (Elf32_Ehdr *)page_addr; + Elf32_Phdr *phdr; + int i; + + /* only supports phdr that fits in one page */ + if (ehdr->e_phnum > + (PAGE_SIZE - sizeof(Elf32_Ehdr)) / sizeof(Elf32_Phdr)) + return -EINVAL; + + phdr = (Elf32_Phdr *)(page_addr + sizeof(Elf32_Ehdr)); + + for (i = 0; i < ehdr->e_phnum; ++i) { + if (phdr[i].p_type == PT_NOTE && + !parse_build_id(page_addr, build_id, size, + page_addr + phdr[i].p_offset, + phdr[i].p_filesz)) + return 0; + } + return -EINVAL; +} + +/* Parse build ID from 64-bit ELF */ +static int get_build_id_64(const void *page_addr, unsigned char *build_id, + __u32 *size) +{ + Elf64_Ehdr *ehdr = (Elf64_Ehdr *)page_addr; + Elf64_Phdr *phdr; + int i; + + /* only supports phdr that fits in one page */ + if (ehdr->e_phnum > + (PAGE_SIZE - sizeof(Elf64_Ehdr)) / sizeof(Elf64_Phdr)) + return -EINVAL; + + phdr = (Elf64_Phdr *)(page_addr + sizeof(Elf64_Ehdr)); + + for (i = 0; i < ehdr->e_phnum; ++i) { + if (phdr[i].p_type == PT_NOTE && + !parse_build_id(page_addr, build_id, size, + page_addr + phdr[i].p_offset, + phdr[i].p_filesz)) + return 0; + } + return -EINVAL; +} + +/* + * Parse build ID of ELF file mapped to vma + * @vma: vma object + * @build_id: buffer to store build id, at least BUILD_ID_SIZE long + * @size: returns actual build id size in case of success + * + * Return: 0 on success, -EINVAL otherwise + */ +int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, + __u32 *size) +{ + Elf32_Ehdr *ehdr; + struct page *page; + void *page_addr; + int ret; + + /* only works for page backed storage */ + if (!vma->vm_file) + return -EINVAL; + + page = find_get_page(vma->vm_file->f_mapping, 0); + if (!page) + return -EFAULT; /* page not mapped */ + + ret = -EINVAL; + page_addr = kmap_atomic(page); + ehdr = (Elf32_Ehdr *)page_addr; + + /* compare magic x7f "ELF" */ + if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) + goto out; + + /* only support executable file and shared object file */ + if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) + goto out; + + if (ehdr->e_ident[EI_CLASS] == ELFCLASS32) + ret = get_build_id_32(page_addr, build_id, size); + else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64) + ret = get_build_id_64(page_addr, build_id, size); +out: + kunmap_atomic(page_addr); + put_page(page); + return ret; +} + +/** + * build_id_parse_buf - Get build ID from a buffer + * @buf: Elf note section(s) to parse + * @buf_size: Size of @buf in bytes + * @build_id: Build ID parsed from @buf, at least BUILD_ID_SIZE_MAX long + * + * Return: 0 on success, -EINVAL otherwise + */ +int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size) +{ + return parse_build_id_buf(build_id, NULL, buf, buf_size); +} + +#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) || IS_ENABLED(CONFIG_CRASH_CORE) +unsigned char vmlinux_build_id[BUILD_ID_SIZE_MAX] __ro_after_init; + +/** + * init_vmlinux_build_id - Compute and stash the running kernel's build ID + */ +void __init init_vmlinux_build_id(void) +{ + extern const void __start_notes __weak; + extern const void __stop_notes __weak; + unsigned int size = &__stop_notes - &__start_notes; + + build_id_parse_buf(&__start_notes, vmlinux_build_id, size); +} +#endif diff --git a/lib/checksum.c b/lib/checksum.c index de032ad96f4a..6860d6b05a17 100644 --- a/lib/checksum.c +++ b/lib/checksum.c @@ -145,37 +145,6 @@ __sum16 ip_compute_csum(const void *buff, int len) } EXPORT_SYMBOL(ip_compute_csum); -/* - * copy from fs while checksumming, otherwise like csum_partial - */ -__wsum -csum_partial_copy_from_user(const void __user *src, void *dst, int len, - __wsum sum, int *csum_err) -{ - int missing; - - missing = __copy_from_user(dst, src, len); - if (missing) { - memset(dst + len - missing, 0, missing); - *csum_err = -EFAULT; - } else - *csum_err = 0; - - return csum_partial(dst, len, sum); -} -EXPORT_SYMBOL(csum_partial_copy_from_user); - -/* - * copy from ds while checksumming, otherwise like csum_partial - */ -__wsum -csum_partial_copy(const void *src, void *dst, int len, __wsum sum) -{ - memcpy(dst, src, len); - return csum_partial(dst, len, sum); -} -EXPORT_SYMBOL(csum_partial_copy); - #ifndef csum_tcpudp_nofold static inline u32 from64to32(u64 x) { diff --git a/lib/cmdline.c b/lib/cmdline.c index fbb9981a04a4..5546bf588780 100644 --- a/lib/cmdline.c +++ b/lib/cmdline.c @@ -35,25 +35,37 @@ static int get_range(char **str, int *pint, int n) /** * get_option - Parse integer from an option string * @str: option string - * @pint: (output) integer value parsed from @str + * @pint: (optional output) integer value parsed from @str * * Read an int from an option string; if available accept a subsequent * comma as well. * + * When @pint is NULL the function can be used as a validator of + * the current option in the string. + * * Return values: * 0 - no int in string * 1 - int found, no subsequent comma * 2 - int found including a subsequent comma * 3 - hyphen found to denote a range + * + * Leading hyphen without integer is no integer case, but we consume it + * for the sake of simplification. */ int get_option(char **str, int *pint) { char *cur = *str; + int value; if (!cur || !(*cur)) return 0; - *pint = simple_strtol(cur, str, 0); + if (*cur == '-') + value = -simple_strtoull(++cur, str, 0); + else + value = simple_strtoull(cur, str, 0); + if (pint) + *pint = value; if (cur == *str) return 0; if (**str == ',') { @@ -71,7 +83,7 @@ EXPORT_SYMBOL(get_option); * get_options - Parse a string into a list of integers * @str: String to be parsed * @nints: size of integer array - * @ints: integer array + * @ints: integer array (must have room for at least one element) * * This function parses a string containing a comma-separated * list of integers, a hyphen-separated range of _positive_ integers, @@ -79,6 +91,14 @@ EXPORT_SYMBOL(get_option); * full, or when no more numbers can be retrieved from the * string. * + * When @nints is 0, the function just validates the given @str and + * returns the amount of parseable integers as described below. + * + * Returns: + * + * The first element is filled by the number of collected integers + * in the range. The rest is what was parsed from the @str. + * * Return value is the character in the string which caused * the parse to end (typically a null terminator, if @str is * completely parseable). @@ -86,15 +106,20 @@ EXPORT_SYMBOL(get_option); char *get_options(const char *str, int nints, int *ints) { + bool validate = (nints == 0); int res, i = 1; - while (i < nints) { - res = get_option((char **)&str, ints + i); + while (i < nints || validate) { + int *pint = validate ? ints : ints + i; + + res = get_option((char **)&str, pint); if (res == 0) break; if (res == 3) { + int n = validate ? 0 : nints - i; int range_nums; - range_nums = get_range((char **)&str, ints + i, nints - i); + + range_nums = get_range((char **)&str, pint, n); if (range_nums < 0) break; /* @@ -132,27 +157,28 @@ unsigned long long memparse(const char *ptr, char **retptr) case 'E': case 'e': ret <<= 10; - /* fall through */ + fallthrough; case 'P': case 'p': ret <<= 10; - /* fall through */ + fallthrough; case 'T': case 't': ret <<= 10; - /* fall through */ + fallthrough; case 'G': case 'g': ret <<= 10; - /* fall through */ + fallthrough; case 'M': case 'm': ret <<= 10; - /* fall through */ + fallthrough; case 'K': case 'k': ret <<= 10; endptr++; + fallthrough; default: break; } @@ -202,7 +228,6 @@ char *next_arg(char *args, char **param, char **val) { unsigned int i, equals = 0; int in_quote = 0, quoted = 0; - char *next; if (*args == '"') { args++; @@ -240,10 +265,11 @@ char *next_arg(char *args, char **param, char **val) if (args[i]) { args[i] = '\0'; - next = args + i + 1; + args += i + 1; } else - next = args + i; + args += i; /* Chew up trailing spaces. */ - return skip_spaces(next); + return skip_spaces(args); } +EXPORT_SYMBOL(next_arg); diff --git a/lib/cmdline_kunit.c b/lib/cmdline_kunit.c new file mode 100644 index 000000000000..a72a2c16066e --- /dev/null +++ b/lib/cmdline_kunit.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Test cases for API provided by cmdline.c + */ + +#include <kunit/test.h> +#include <linux/kernel.h> +#include <linux/random.h> +#include <linux/string.h> + +static const char *cmdline_test_strings[] = { + "\"\"", "" , "=" , "\"-", "," , "-," , ",-" , "-" , + "+," , "--", ",,", "''" , "\"\",", "\",\"", "-\"\"", "\"", +}; + +static const int cmdline_test_values[] = { + 1, 1, 1, 1, 2, 3, 2, 3, + 1, 3, 2, 1, 1, 1, 3, 1, +}; + +static_assert(ARRAY_SIZE(cmdline_test_strings) == ARRAY_SIZE(cmdline_test_values)); + +static const char *cmdline_test_range_strings[] = { + "-7" , "--7" , "-1-2" , "7--9", + "7-" , "-7--9", "7-9," , "9-7" , + "5-a", "a-5" , "5-8" , ",8-5", + "+,1", "-,4" , "-3,0-1,6", "4,-" , + " +2", " -9" , "0-1,-3,6", "- 9" , +}; + +static const int cmdline_test_range_values[][16] = { + { 1, -7, }, { 0, -0, }, { 4, -1, 0, +1, 2, }, { 0, 7, }, + { 0, +7, }, { 0, -7, }, { 3, +7, 8, +9, 0, }, { 0, 9, }, + { 0, +5, }, { 0, -0, }, { 4, +5, 6, +7, 8, }, { 0, 0, }, + { 0, +0, }, { 0, -0, }, { 4, -3, 0, +1, 6, }, { 1, 4, }, + { 0, +0, }, { 0, -0, }, { 4, +0, 1, -3, 6, }, { 0, 0, }, +}; + +static_assert(ARRAY_SIZE(cmdline_test_range_strings) == ARRAY_SIZE(cmdline_test_range_values)); + +static void cmdline_do_one_test(struct kunit *test, const char *in, int rc, int offset) +{ + const char *fmt = "Pattern: %s"; + const char *out = in; + int dummy; + int ret; + + ret = get_option((char **)&out, &dummy); + + KUNIT_EXPECT_EQ_MSG(test, ret, rc, fmt, in); + KUNIT_EXPECT_PTR_EQ_MSG(test, out, in + offset, fmt, in); +} + +static void cmdline_test_noint(struct kunit *test) +{ + unsigned int i = 0; + + do { + const char *str = cmdline_test_strings[i]; + int rc = 0; + int offset; + + /* Only first and leading '-' will advance the pointer */ + offset = !!(*str == '-'); + cmdline_do_one_test(test, str, rc, offset); + } while (++i < ARRAY_SIZE(cmdline_test_strings)); +} + +static void cmdline_test_lead_int(struct kunit *test) +{ + unsigned int i = 0; + char in[32]; + + do { + const char *str = cmdline_test_strings[i]; + int rc = cmdline_test_values[i]; + int offset; + + sprintf(in, "%u%s", get_random_int() % 256, str); + /* Only first '-' after the number will advance the pointer */ + offset = strlen(in) - strlen(str) + !!(rc == 2); + cmdline_do_one_test(test, in, rc, offset); + } while (++i < ARRAY_SIZE(cmdline_test_strings)); +} + +static void cmdline_test_tail_int(struct kunit *test) +{ + unsigned int i = 0; + char in[32]; + + do { + const char *str = cmdline_test_strings[i]; + /* When "" or "-" the result will be valid integer */ + int rc = strcmp(str, "") ? (strcmp(str, "-") ? 0 : 1) : 1; + int offset; + + sprintf(in, "%s%u", str, get_random_int() % 256); + /* + * Only first and leading '-' not followed by integer + * will advance the pointer. + */ + offset = rc ? strlen(in) : !!(*str == '-'); + cmdline_do_one_test(test, in, rc, offset); + } while (++i < ARRAY_SIZE(cmdline_test_strings)); +} + +static void cmdline_do_one_range_test(struct kunit *test, const char *in, + unsigned int n, const int *e) +{ + unsigned int i; + int r[16]; + int *p; + + memset(r, 0, sizeof(r)); + get_options(in, ARRAY_SIZE(r), r); + KUNIT_EXPECT_EQ_MSG(test, r[0], e[0], "in test %u (parsed) expected %d numbers, got %d", + n, e[0], r[0]); + for (i = 1; i < ARRAY_SIZE(r); i++) + KUNIT_EXPECT_EQ_MSG(test, r[i], e[i], "in test %u at %u", n, i); + + memset(r, 0, sizeof(r)); + get_options(in, 0, r); + KUNIT_EXPECT_EQ_MSG(test, r[0], e[0], "in test %u (validated) expected %d numbers, got %d", + n, e[0], r[0]); + + p = memchr_inv(&r[1], 0, sizeof(r) - sizeof(r[0])); + KUNIT_EXPECT_PTR_EQ_MSG(test, p, NULL, "in test %u at %u out of bound", n, p - r); +} + +static void cmdline_test_range(struct kunit *test) +{ + unsigned int i = 0; + + do { + const char *str = cmdline_test_range_strings[i]; + const int *e = cmdline_test_range_values[i]; + + cmdline_do_one_range_test(test, str, i, e); + } while (++i < ARRAY_SIZE(cmdline_test_range_strings)); +} + +static struct kunit_case cmdline_test_cases[] = { + KUNIT_CASE(cmdline_test_noint), + KUNIT_CASE(cmdline_test_lead_int), + KUNIT_CASE(cmdline_test_tail_int), + KUNIT_CASE(cmdline_test_range), + {} +}; + +static struct kunit_suite cmdline_test_suite = { + .name = "cmdline", + .test_cases = cmdline_test_cases, +}; +kunit_test_suite(cmdline_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/lib/compat_audit.c b/lib/compat_audit.c index 77eabad69b4a..3d6b8996f027 100644 --- a/lib/compat_audit.c +++ b/lib/compat_audit.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/init.h> #include <linux/types.h> +#include <linux/audit_arch.h> #include <asm/unistd32.h> unsigned compat_dir_class[] = { @@ -33,19 +34,23 @@ int audit_classify_compat_syscall(int abi, unsigned syscall) switch (syscall) { #ifdef __NR_open case __NR_open: - return 2; + return AUDITSC_OPEN; #endif #ifdef __NR_openat case __NR_openat: - return 3; + return AUDITSC_OPENAT; #endif #ifdef __NR_socketcall case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; #endif case __NR_execve: - return 5; + return AUDITSC_EXECVE; +#ifdef __NR_openat2 + case __NR_openat2: + return AUDITSC_OPENAT2; +#endif default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/lib/cpumask.c b/lib/cpumask.c index 0cb672eb107c..a971a82d2f43 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -188,7 +188,7 @@ EXPORT_SYMBOL(free_cpumask_var); */ void __init free_bootmem_cpumask_var(cpumask_var_t mask) { - memblock_free_early(__pa(mask), cpumask_size()); + memblock_free(mask, cpumask_size()); } #endif @@ -232,3 +232,50 @@ unsigned int cpumask_local_spread(unsigned int i, int node) BUG(); } EXPORT_SYMBOL(cpumask_local_spread); + +static DEFINE_PER_CPU(int, distribute_cpu_mask_prev); + +/** + * Returns an arbitrary cpu within srcp1 & srcp2. + * + * Iterated calls using the same srcp1 and srcp2 will be distributed within + * their intersection. + * + * Returns >= nr_cpu_ids if the intersection is empty. + */ +int cpumask_any_and_distribute(const struct cpumask *src1p, + const struct cpumask *src2p) +{ + int next, prev; + + /* NOTE: our first selection will skip 0. */ + prev = __this_cpu_read(distribute_cpu_mask_prev); + + next = cpumask_next_and(prev, src1p, src2p); + if (next >= nr_cpu_ids) + next = cpumask_first_and(src1p, src2p); + + if (next < nr_cpu_ids) + __this_cpu_write(distribute_cpu_mask_prev, next); + + return next; +} +EXPORT_SYMBOL(cpumask_any_and_distribute); + +int cpumask_any_distribute(const struct cpumask *srcp) +{ + int next, prev; + + /* NOTE: our first selection will skip 0. */ + prev = __this_cpu_read(distribute_cpu_mask_prev); + + next = cpumask_next(prev, srcp); + if (next >= nr_cpu_ids) + next = cpumask_first(srcp); + + if (next < nr_cpu_ids) + __this_cpu_write(distribute_cpu_mask_prev, next); + + return next; +} +EXPORT_SYMBOL(cpumask_any_distribute); diff --git a/lib/crc-t10dif.c b/lib/crc-t10dif.c index 8cc01a603416..1ed2ed487097 100644 --- a/lib/crc-t10dif.c +++ b/lib/crc-t10dif.c @@ -17,64 +17,69 @@ #include <linux/notifier.h> static struct crypto_shash __rcu *crct10dif_tfm; -static struct static_key crct10dif_fallback __read_mostly; +static DEFINE_STATIC_KEY_TRUE(crct10dif_fallback); static DEFINE_MUTEX(crc_t10dif_mutex); +static struct work_struct crct10dif_rehash_work; -static int crc_t10dif_rehash(struct notifier_block *self, unsigned long val, void *data) +static int crc_t10dif_notify(struct notifier_block *self, unsigned long val, void *data) { struct crypto_alg *alg = data; - struct crypto_shash *new, *old; if (val != CRYPTO_MSG_ALG_LOADED || - static_key_false(&crct10dif_fallback) || - strncmp(alg->cra_name, CRC_T10DIF_STRING, strlen(CRC_T10DIF_STRING))) - return 0; + strcmp(alg->cra_name, CRC_T10DIF_STRING)) + return NOTIFY_DONE; + + schedule_work(&crct10dif_rehash_work); + return NOTIFY_OK; +} + +static void crc_t10dif_rehash(struct work_struct *work) +{ + struct crypto_shash *new, *old; mutex_lock(&crc_t10dif_mutex); old = rcu_dereference_protected(crct10dif_tfm, lockdep_is_held(&crc_t10dif_mutex)); - if (!old) { - mutex_unlock(&crc_t10dif_mutex); - return 0; - } - new = crypto_alloc_shash("crct10dif", 0, 0); + new = crypto_alloc_shash(CRC_T10DIF_STRING, 0, 0); if (IS_ERR(new)) { mutex_unlock(&crc_t10dif_mutex); - return 0; + return; } rcu_assign_pointer(crct10dif_tfm, new); mutex_unlock(&crc_t10dif_mutex); - synchronize_rcu(); - crypto_free_shash(old); - return 0; + if (old) { + synchronize_rcu(); + crypto_free_shash(old); + } else { + static_branch_disable(&crct10dif_fallback); + } } static struct notifier_block crc_t10dif_nb = { - .notifier_call = crc_t10dif_rehash, + .notifier_call = crc_t10dif_notify, }; __u16 crc_t10dif_update(__u16 crc, const unsigned char *buffer, size_t len) { struct { struct shash_desc shash; - char ctx[2]; + __u16 crc; } desc; int err; - if (static_key_false(&crct10dif_fallback)) + if (static_branch_unlikely(&crct10dif_fallback)) return crc_t10dif_generic(crc, buffer, len); rcu_read_lock(); desc.shash.tfm = rcu_dereference(crct10dif_tfm); - *(__u16 *)desc.ctx = crc; - + desc.crc = crc; err = crypto_shash_update(&desc.shash, buffer, len); rcu_read_unlock(); BUG_ON(err); - return *(__u16 *)desc.ctx; + return desc.crc; } EXPORT_SYMBOL(crc_t10dif_update); @@ -86,19 +91,17 @@ EXPORT_SYMBOL(crc_t10dif); static int __init crc_t10dif_mod_init(void) { + INIT_WORK(&crct10dif_rehash_work, crc_t10dif_rehash); crypto_register_notifier(&crc_t10dif_nb); - crct10dif_tfm = crypto_alloc_shash("crct10dif", 0, 0); - if (IS_ERR(crct10dif_tfm)) { - static_key_slow_inc(&crct10dif_fallback); - crct10dif_tfm = NULL; - } + crc_t10dif_rehash(&crct10dif_rehash_work); return 0; } static void __exit crc_t10dif_mod_fini(void) { crypto_unregister_notifier(&crc_t10dif_nb); - crypto_free_shash(crct10dif_tfm); + cancel_work_sync(&crct10dif_rehash_work); + crypto_free_shash(rcu_dereference_protected(crct10dif_tfm, 1)); } module_init(crc_t10dif_mod_init); @@ -106,15 +109,23 @@ module_exit(crc_t10dif_mod_fini); static int crc_t10dif_transform_show(char *buffer, const struct kernel_param *kp) { - if (static_key_false(&crct10dif_fallback)) + struct crypto_shash *tfm; + int len; + + if (static_branch_unlikely(&crct10dif_fallback)) return sprintf(buffer, "fallback\n"); - return sprintf(buffer, "%s\n", - crypto_tfm_alg_driver_name(crypto_shash_tfm(crct10dif_tfm))); + rcu_read_lock(); + tfm = rcu_dereference(crct10dif_tfm); + len = snprintf(buffer, PAGE_SIZE, "%s\n", + crypto_shash_driver_name(tfm)); + rcu_read_unlock(); + + return len; } -module_param_call(transform, NULL, crc_t10dif_transform_show, NULL, 0644); +module_param_call(transform, NULL, crc_t10dif_transform_show, NULL, 0444); -MODULE_DESCRIPTION("T10 DIF CRC calculation"); +MODULE_DESCRIPTION("T10 DIF CRC calculation (library API)"); MODULE_LICENSE("GPL"); MODULE_SOFTDEP("pre: crct10dif"); diff --git a/lib/crc32.c b/lib/crc32.c index 4a20455d1f61..2a68dfd3b96c 100644 --- a/lib/crc32.c +++ b/lib/crc32.c @@ -24,7 +24,7 @@ * Version 2. See the file COPYING for more details. */ -/* see: Documentation/crc32.txt for a description of algorithms */ +/* see: Documentation/staging/crc32.rst for a description of algorithms */ #include <linux/crc32.h> #include <linux/crc32poly.h> @@ -331,7 +331,7 @@ static inline u32 __pure crc32_be_generic(u32 crc, unsigned char const *p, return crc; } -#if CRC_LE_BITS == 1 +#if CRC_BE_BITS == 1 u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) { return crc32_be_generic(crc, p, len, NULL, CRC32_POLY_BE); diff --git a/lib/crc32test.c b/lib/crc32test.c index 97d6a57cefcc..61ddce2cff77 100644 --- a/lib/crc32test.c +++ b/lib/crc32test.c @@ -683,7 +683,6 @@ static int __init crc32c_test(void) /* reduce OS noise */ local_irq_save(flags); - local_irq_disable(); nsec = ktime_get_ns(); for (i = 0; i < 100; i++) { @@ -694,7 +693,6 @@ static int __init crc32c_test(void) nsec = ktime_get_ns() - nsec; local_irq_restore(flags); - local_irq_enable(); pr_info("crc32c: CRC_LE_BITS = %d\n", CRC_LE_BITS); @@ -768,7 +766,6 @@ static int __init crc32_test(void) /* reduce OS noise */ local_irq_save(flags); - local_irq_disable(); nsec = ktime_get_ns(); for (i = 0; i < 100; i++) { @@ -783,7 +780,6 @@ static int __init crc32_test(void) nsec = ktime_get_ns() - nsec; local_irq_restore(flags); - local_irq_enable(); pr_info("crc32: CRC_LE_BITS = %d, CRC_BE BITS = %d\n", CRC_LE_BITS, CRC_BE_BITS); diff --git a/lib/crc64.c b/lib/crc64.c index 0ef8ae6ac047..9f852a89ee2a 100644 --- a/lib/crc64.c +++ b/lib/crc64.c @@ -4,7 +4,7 @@ * * This is a basic crc64 implementation following ECMA-182 specification, * which can be found from, - * http://www.ecma-international.org/publications/standards/Ecma-182.htm + * https://www.ecma-international.org/publications/standards/Ecma-182.htm * * Dr. Ross N. Williams has a great document to introduce the idea of CRC * algorithm, here the CRC64 code is also inspired by the table-driven @@ -28,6 +28,7 @@ #include <linux/module.h> #include <linux/types.h> +#include <linux/crc64.h> #include "crc64table.h" MODULE_DESCRIPTION("CRC64 calculations"); @@ -36,7 +37,7 @@ MODULE_LICENSE("GPL v2"); /** * crc64_be - Calculate bitwise big-endian ECMA-182 CRC64 * @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation, - or the previous crc64 value if computing incrementally. + * or the previous crc64 value if computing incrementally. * @p: pointer to buffer over which CRC64 is run * @len: length of buffer @p */ diff --git a/lib/crc7.c b/lib/crc7.c index 6a848d73e804..3848e313b722 100644 --- a/lib/crc7.c +++ b/lib/crc7.c @@ -51,7 +51,7 @@ const u8 crc7_be_syndrome_table[256] = { EXPORT_SYMBOL(crc7_be_syndrome_table); /** - * crc7 - update the CRC7 for the data buffer + * crc7_be - update the CRC7 for the data buffer * @crc: previous CRC7 value * @buffer: data pointer * @len: number of bytes in the buffer diff --git a/lib/crc8.c b/lib/crc8.c index 595a5a75e3cd..1ad8e501d9b6 100644 --- a/lib/crc8.c +++ b/lib/crc8.c @@ -71,7 +71,7 @@ EXPORT_SYMBOL(crc8_populate_lsb); * @nbytes: number of bytes in data buffer. * @crc: previous returned crc8 value. */ -u8 crc8(const u8 table[CRC8_TABLE_SIZE], u8 *pdata, size_t nbytes, u8 crc) +u8 crc8(const u8 table[CRC8_TABLE_SIZE], const u8 *pdata, size_t nbytes, u8 crc) { /* loop over the buffer data */ while (nbytes-- > 0) diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 0b2c4fce26d9..545ccbddf6a1 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -90,7 +90,7 @@ config CRYPTO_LIB_DES config CRYPTO_LIB_POLY1305_RSIZE int default 2 if MIPS - default 4 if X86_64 + default 11 if X86_64 default 9 if ARM || ARM64 default 1 @@ -128,3 +128,6 @@ config CRYPTO_LIB_CHACHA20POLY1305 config CRYPTO_LIB_SHA256 tristate + +config CRYPTO_LIB_SM4 + tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 34a701ab8b92..73205ed269ba 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -19,21 +19,30 @@ libblake2s-y += blake2s.o obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o libchacha20poly1305-y += chacha20poly1305.o -obj-$(CONFIG_CRYPTO_LIB_CURVE25519_GENERIC) += libcurve25519.o -libcurve25519-y := curve25519-fiat32.o -libcurve25519-$(CONFIG_ARCH_SUPPORTS_INT128) := curve25519-hacl64.o +obj-$(CONFIG_CRYPTO_LIB_CURVE25519_GENERIC) += libcurve25519-generic.o +libcurve25519-generic-y := curve25519-fiat32.o +libcurve25519-generic-$(CONFIG_ARCH_SUPPORTS_INT128) := curve25519-hacl64.o +libcurve25519-generic-y += curve25519-generic.o + +obj-$(CONFIG_CRYPTO_LIB_CURVE25519) += libcurve25519.o libcurve25519-y += curve25519.o obj-$(CONFIG_CRYPTO_LIB_DES) += libdes.o libdes-y := des.o obj-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += libpoly1305.o -libpoly1305-y := poly1305.o +libpoly1305-y := poly1305-donna32.o +libpoly1305-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o +libpoly1305-y += poly1305.o obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o libsha256-y := sha256.o +obj-$(CONFIG_CRYPTO_LIB_SM4) += libsm4.o +libsm4-y := sm4.o + ifneq ($(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS),y) libblake2s-y += blake2s-selftest.o libchacha20poly1305-y += chacha20poly1305-selftest.o +libcurve25519-y += curve25519-selftest.o endif diff --git a/lib/crypto/blake2s-selftest.c b/lib/crypto/blake2s-selftest.c index 79ef404a990d..5d9ea53be973 100644 --- a/lib/crypto/blake2s-selftest.c +++ b/lib/crypto/blake2s-selftest.c @@ -3,7 +3,7 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. */ -#include <crypto/blake2s.h> +#include <crypto/internal/blake2s.h> #include <linux/string.h> /* diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c index 41025a30c524..4055aa593ec4 100644 --- a/lib/crypto/blake2s.c +++ b/lib/crypto/blake2s.c @@ -15,57 +15,23 @@ #include <linux/module.h> #include <linux/init.h> #include <linux/bug.h> -#include <asm/unaligned.h> -bool blake2s_selftest(void); +#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_BLAKE2S) +# define blake2s_compress blake2s_compress_arch +#else +# define blake2s_compress blake2s_compress_generic +#endif void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen) { - const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; - - if (unlikely(!inlen)) - return; - if (inlen > fill) { - memcpy(state->buf + state->buflen, in, fill); - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_BLAKE2S)) - blake2s_compress_arch(state, state->buf, 1, - BLAKE2S_BLOCK_SIZE); - else - blake2s_compress_generic(state, state->buf, 1, - BLAKE2S_BLOCK_SIZE); - state->buflen = 0; - in += fill; - inlen -= fill; - } - if (inlen > BLAKE2S_BLOCK_SIZE) { - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); - /* Hash one less (full) block than strictly possible */ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_BLAKE2S)) - blake2s_compress_arch(state, in, nblocks - 1, - BLAKE2S_BLOCK_SIZE); - else - blake2s_compress_generic(state, in, nblocks - 1, - BLAKE2S_BLOCK_SIZE); - in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); - inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); - } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; + __blake2s_update(state, in, inlen, blake2s_compress); } EXPORT_SYMBOL(blake2s_update); void blake2s_final(struct blake2s_state *state, u8 *out) { WARN_ON(IS_ENABLED(DEBUG) && !out); - blake2s_set_lastblock(state); - memset(state->buf + state->buflen, 0, - BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_BLAKE2S)) - blake2s_compress_arch(state, state->buf, 1, state->buflen); - else - blake2s_compress_generic(state, state->buf, 1, state->buflen); - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - memcpy(out, state->h, state->outlen); + __blake2s_final(state, out, blake2s_compress); memzero_explicit(state, sizeof(*state)); } EXPORT_SYMBOL(blake2s_final); @@ -107,7 +73,7 @@ void blake2s256_hmac(u8 *out, const u8 *in, const u8 *key, const size_t inlen, } EXPORT_SYMBOL(blake2s256_hmac); -static int __init mod_init(void) +static int __init blake2s_mod_init(void) { if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && WARN_ON(!blake2s_selftest())) @@ -115,12 +81,12 @@ static int __init mod_init(void) return 0; } -static void __exit mod_exit(void) +static void __exit blake2s_mod_exit(void) { } -module_init(mod_init); -module_exit(mod_exit); +module_init(blake2s_mod_init); +module_exit(blake2s_mod_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("BLAKE2s hash function"); MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index 65ead6b0c7e0..b748fd3d256e 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -10,7 +10,6 @@ #include <linux/export.h> #include <linux/bitops.h> #include <linux/string.h> -#include <linux/cryptohash.h> #include <asm/unaligned.h> #include <crypto/chacha.h> @@ -65,7 +64,7 @@ static void chacha_permute(u32 *x, int nrounds) } /** - * chacha_block - generate one keystream block and increment block counter + * chacha_block_generic - generate one keystream block and increment block counter * @state: input state matrix (16 32-bit words) * @stream: output keystream block (64 bytes) * @nrounds: number of rounds (20 or 12; 20 is recommended) @@ -93,7 +92,7 @@ EXPORT_SYMBOL(chacha_block_generic); /** * hchacha_block_generic - abbreviated ChaCha core, for XChaCha * @state: input state matrix (16 32-bit words) - * @out: output (8 32-bit words) + * @stream: output (8 32-bit words) * @nrounds: number of rounds (20 or 12; 20 is recommended) * * HChaCha is the ChaCha equivalent of HSalsa and is an intermediate step diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/chacha20poly1305-selftest.c index 465de46dbdef..fa43deda2660 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/chacha20poly1305-selftest.c @@ -4,6 +4,7 @@ */ #include <crypto/chacha20poly1305.h> +#include <crypto/chacha.h> #include <crypto/poly1305.h> #include <asm/unaligned.h> @@ -1926,6 +1927,1104 @@ static const u8 enc_key012[] __initconst = { 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; +/* wycheproof - rfc7539 */ +static const u8 enc_input013[] __initconst = { + 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, + 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x61, 0x73, + 0x73, 0x20, 0x6f, 0x66, 0x20, 0x27, 0x39, 0x39, + 0x3a, 0x20, 0x49, 0x66, 0x20, 0x49, 0x20, 0x63, + 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6f, 0x66, 0x66, + 0x65, 0x72, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6f, + 0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x65, 0x20, + 0x74, 0x69, 0x70, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x66, 0x75, 0x74, 0x75, + 0x72, 0x65, 0x2c, 0x20, 0x73, 0x75, 0x6e, 0x73, + 0x63, 0x72, 0x65, 0x65, 0x6e, 0x20, 0x77, 0x6f, + 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, + 0x74, 0x2e +}; +static const u8 enc_output013[] __initconst = { + 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, + 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, + 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, + 0xa9, 0xe2, 0xb5, 0xa7, 0x36, 0xee, 0x62, 0xd6, + 0x3d, 0xbe, 0xa4, 0x5e, 0x8c, 0xa9, 0x67, 0x12, + 0x82, 0xfa, 0xfb, 0x69, 0xda, 0x92, 0x72, 0x8b, + 0x1a, 0x71, 0xde, 0x0a, 0x9e, 0x06, 0x0b, 0x29, + 0x05, 0xd6, 0xa5, 0xb6, 0x7e, 0xcd, 0x3b, 0x36, + 0x92, 0xdd, 0xbd, 0x7f, 0x2d, 0x77, 0x8b, 0x8c, + 0x98, 0x03, 0xae, 0xe3, 0x28, 0x09, 0x1b, 0x58, + 0xfa, 0xb3, 0x24, 0xe4, 0xfa, 0xd6, 0x75, 0x94, + 0x55, 0x85, 0x80, 0x8b, 0x48, 0x31, 0xd7, 0xbc, + 0x3f, 0xf4, 0xde, 0xf0, 0x8e, 0x4b, 0x7a, 0x9d, + 0xe5, 0x76, 0xd2, 0x65, 0x86, 0xce, 0xc6, 0x4b, + 0x61, 0x16, 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, + 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, + 0x06, 0x91 +}; +static const u8 enc_assoc013[] __initconst = { + 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7 +}; +static const u8 enc_nonce013[] __initconst = { + 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47 +}; +static const u8 enc_key013[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input014[] __initconst = { }; +static const u8 enc_output014[] __initconst = { + 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, + 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d +}; +static const u8 enc_assoc014[] __initconst = { }; +static const u8 enc_nonce014[] __initconst = { + 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, + 0xea, 0x12, 0x37, 0x9d +}; +static const u8 enc_key014[] __initconst = { + 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, + 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, + 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, + 0x42, 0x6e, 0x11, 0x40, 0x9c, 0x09, 0xb9, 0xb0 +}; + +/* wycheproof - misc */ +static const u8 enc_input015[] __initconst = { }; +static const u8 enc_output015[] __initconst = { + 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, + 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 +}; +static const u8 enc_assoc015[] __initconst = { + 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 +}; +static const u8 enc_nonce015[] __initconst = { + 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, + 0xa3, 0xc6, 0xf6, 0x89 +}; +static const u8 enc_key015[] __initconst = { + 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, + 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, + 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, + 0x91, 0xdb, 0xb9, 0x57, 0x7a, 0x25, 0x07, 0x42 +}; + +/* wycheproof - misc */ +static const u8 enc_input016[] __initconst = { + 0x2a +}; +static const u8 enc_output016[] __initconst = { + 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, + 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, + 0x22 +}; +static const u8 enc_assoc016[] __initconst = { }; +static const u8 enc_nonce016[] __initconst = { + 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, + 0xee, 0xab, 0x60, 0xf1 +}; +static const u8 enc_key016[] __initconst = { + 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, + 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, + 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, + 0xb6, 0xae, 0xf4, 0x91, 0x6a, 0x76, 0x62, 0x73 +}; + +/* wycheproof - misc */ +static const u8 enc_input017[] __initconst = { + 0x51 +}; +static const u8 enc_output017[] __initconst = { + 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, + 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, + 0xb9 +}; +static const u8 enc_assoc017[] __initconst = { + 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 +}; +static const u8 enc_nonce017[] __initconst = { + 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, + 0x78, 0x2f, 0x44, 0x03 +}; +static const u8 enc_key017[] __initconst = { + 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, + 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, + 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, + 0x5e, 0x4b, 0x70, 0xf0, 0xfd, 0xa6, 0x20, 0xee +}; + +/* wycheproof - misc */ +static const u8 enc_input018[] __initconst = { + 0x5c, 0x60 +}; +static const u8 enc_output018[] __initconst = { + 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, + 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, + 0xe5, 0xe2 +}; +static const u8 enc_assoc018[] __initconst = { }; +static const u8 enc_nonce018[] __initconst = { + 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, + 0x7e, 0x03, 0xf2, 0xdb +}; +static const u8 enc_key018[] __initconst = { + 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, + 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, + 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, + 0xe6, 0xc0, 0x25, 0xfd, 0x9b, 0xa4, 0xc1, 0x00 +}; + +/* wycheproof - misc */ +static const u8 enc_input019[] __initconst = { + 0xdd, 0xf2 +}; +static const u8 enc_output019[] __initconst = { + 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, + 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, + 0x37, 0xa2 +}; +static const u8 enc_assoc019[] __initconst = { + 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf +}; +static const u8 enc_nonce019[] __initconst = { + 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, + 0xb6, 0x04, 0x0a, 0xc6 +}; +static const u8 enc_key019[] __initconst = { + 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, + 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, + 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, + 0xe5, 0x71, 0x8f, 0xe0, 0xd7, 0xab, 0x1a, 0x58 +}; + +/* wycheproof - misc */ +static const u8 enc_input020[] __initconst = { + 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 +}; +static const u8 enc_output020[] __initconst = { + 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, + 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, + 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 +}; +static const u8 enc_assoc020[] __initconst = { }; +static const u8 enc_nonce020[] __initconst = { + 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, + 0x6d, 0xc5, 0x5b, 0xb7 +}; +static const u8 enc_key020[] __initconst = { + 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, + 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, + 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, + 0x4a, 0x37, 0x4f, 0xdb, 0xfc, 0x5a, 0xda, 0xc7 +}; + +/* wycheproof - misc */ +static const u8 enc_input021[] __initconst = { + 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 +}; +static const u8 enc_output021[] __initconst = { + 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, + 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, + 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba +}; +static const u8 enc_assoc021[] __initconst = { + 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 +}; +static const u8 enc_nonce021[] __initconst = { + 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, + 0xde, 0x94, 0x83, 0x96 +}; +static const u8 enc_key021[] __initconst = { + 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, + 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, + 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, + 0x52, 0xa7, 0x30, 0xf3, 0x4d, 0x84, 0x0d, 0x7f +}; + +/* wycheproof - misc */ +static const u8 enc_input022[] __initconst = { + 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed +}; +static const u8 enc_output022[] __initconst = { + 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, + 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, + 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 +}; +static const u8 enc_assoc022[] __initconst = { }; +static const u8 enc_nonce022[] __initconst = { + 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, + 0x36, 0x18, 0x23, 0xd3 +}; +static const u8 enc_key022[] __initconst = { + 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, + 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, + 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, + 0xb0, 0x0e, 0x60, 0xe4, 0x8d, 0xb7, 0xe1, 0x52 +}; + +/* wycheproof - misc */ +static const u8 enc_input023[] __initconst = { + 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf +}; +static const u8 enc_output023[] __initconst = { + 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, + 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, + 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 +}; +static const u8 enc_assoc023[] __initconst = { + 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d +}; +static const u8 enc_nonce023[] __initconst = { + 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, + 0x66, 0x48, 0x4a, 0x78 +}; +static const u8 enc_key023[] __initconst = { + 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, + 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, + 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, + 0x4a, 0xbd, 0x56, 0x08, 0xd7, 0xcd, 0xda, 0xdd +}; + +/* wycheproof - misc */ +static const u8 enc_input024[] __initconst = { + 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, + 0x36, 0x8d, 0x14, 0xe0 +}; +static const u8 enc_output024[] __initconst = { + 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, + 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, + 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, + 0x6d, 0xcb, 0x29, 0xb4 +}; +static const u8 enc_assoc024[] __initconst = { }; +static const u8 enc_nonce024[] __initconst = { + 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, + 0x07, 0x53, 0x86, 0x56 +}; +static const u8 enc_key024[] __initconst = { + 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, + 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, + 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, + 0x0b, 0x39, 0xf7, 0xf8, 0xa9, 0xcc, 0x51, 0xbe +}; + +/* wycheproof - misc */ +static const u8 enc_input025[] __initconst = { + 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, + 0x59, 0xb1, 0xa0, 0xda +}; +static const u8 enc_output025[] __initconst = { + 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, + 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, + 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, + 0xcf, 0x05, 0x07, 0x2f +}; +static const u8 enc_assoc025[] __initconst = { + 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 +}; +static const u8 enc_nonce025[] __initconst = { + 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, + 0xd9, 0x06, 0xe9, 0xce +}; +static const u8 enc_key025[] __initconst = { + 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, + 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, + 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, + 0x82, 0x51, 0x18, 0xb0, 0x06, 0xb9, 0x12, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input026[] __initconst = { + 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, + 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 +}; +static const u8 enc_output026[] __initconst = { + 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, + 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, + 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, + 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 +}; +static const u8 enc_assoc026[] __initconst = { }; +static const u8 enc_nonce026[] __initconst = { + 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, + 0xd4, 0x61, 0xd2, 0x3c +}; +static const u8 enc_key026[] __initconst = { + 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, + 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, + 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, + 0x70, 0x9a, 0xf7, 0x55, 0xee, 0xec, 0x6d, 0xcb +}; + +/* wycheproof - misc */ +static const u8 enc_input027[] __initconst = { + 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, + 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 +}; +static const u8 enc_output027[] __initconst = { + 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, + 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, + 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, + 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 +}; +static const u8 enc_assoc027[] __initconst = { + 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 +}; +static const u8 enc_nonce027[] __initconst = { + 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, + 0xc5, 0x03, 0x5b, 0x6a +}; +static const u8 enc_key027[] __initconst = { + 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, + 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, + 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, + 0xbc, 0x8b, 0xc4, 0x89, 0x3a, 0x58, 0x89, 0x4d +}; + +/* wycheproof - misc */ +static const u8 enc_input028[] __initconst = { + 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, + 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 +}; +static const u8 enc_output028[] __initconst = { + 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, + 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, + 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, + 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c +}; +static const u8 enc_assoc028[] __initconst = { }; +static const u8 enc_nonce028[] __initconst = { + 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, + 0x47, 0x40, 0xad, 0x9b +}; +static const u8 enc_key028[] __initconst = { + 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, + 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, + 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, + 0xb0, 0x19, 0xc1, 0x69, 0x9f, 0x87, 0x67, 0x4a +}; + +/* wycheproof - misc */ +static const u8 enc_input029[] __initconst = { + 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, + 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 +}; +static const u8 enc_output029[] __initconst = { + 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, + 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, + 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, + 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d +}; +static const u8 enc_assoc029[] __initconst = { + 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff +}; +static const u8 enc_nonce029[] __initconst = { + 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, + 0x07, 0x1f, 0x52, 0x66 +}; +static const u8 enc_key029[] __initconst = { + 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, + 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, + 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, + 0x77, 0x1c, 0x86, 0x25, 0x71, 0xd2, 0xea, 0x1e +}; + +/* wycheproof - misc */ +static const u8 enc_input030[] __initconst = { + 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, + 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, + 0x1f +}; +static const u8 enc_output030[] __initconst = { + 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, + 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, + 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, + 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, + 0x13 +}; +static const u8 enc_assoc030[] __initconst = { }; +static const u8 enc_nonce030[] __initconst = { + 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, + 0xdc, 0x03, 0x44, 0x5d +}; +static const u8 enc_key030[] __initconst = { + 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, + 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, + 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, + 0x16, 0x61, 0x76, 0x62, 0x96, 0x16, 0xd0, 0x11 +}; + +/* wycheproof - misc */ +static const u8 enc_input031[] __initconst = { + 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, + 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, + 0x9c +}; +static const u8 enc_output031[] __initconst = { + 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, + 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, + 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, + 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, + 0xeb +}; +static const u8 enc_assoc031[] __initconst = { + 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 +}; +static const u8 enc_nonce031[] __initconst = { + 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, + 0x63, 0x3d, 0x99, 0x3d +}; +static const u8 enc_key031[] __initconst = { + 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, + 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, + 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, + 0x17, 0x6d, 0x18, 0x23, 0x06, 0xff, 0xb8, 0x70 +}; + +/* wycheproof - misc */ +static const u8 enc_input032[] __initconst = { + 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, + 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, + 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 +}; +static const u8 enc_output032[] __initconst = { + 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, + 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, + 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, + 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, + 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a +}; +static const u8 enc_assoc032[] __initconst = { }; +static const u8 enc_nonce032[] __initconst = { + 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, + 0x2c, 0x65, 0xf3, 0x1b +}; +static const u8 enc_key032[] __initconst = { + 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, + 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, + 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, + 0x76, 0xbd, 0xcf, 0xdb, 0xc5, 0xce, 0x9c, 0xd7 +}; + +/* wycheproof - misc */ +static const u8 enc_input033[] __initconst = { + 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, + 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, + 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 +}; +static const u8 enc_output033[] __initconst = { + 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, + 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, + 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, + 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, + 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 +}; +static const u8 enc_assoc033[] __initconst = { + 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 +}; +static const u8 enc_nonce033[] __initconst = { + 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, + 0x78, 0x34, 0xed, 0x55 +}; +static const u8 enc_key033[] __initconst = { + 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, + 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, + 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, + 0x0f, 0xae, 0x4b, 0x02, 0x83, 0xd1, 0x8b, 0xe3 +}; + +/* wycheproof - misc */ +static const u8 enc_input034[] __initconst = { + 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, + 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, + 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, + 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 +}; +static const u8 enc_output034[] __initconst = { + 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, + 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, + 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, + 0x89, 0xc9, 0xfd, 0x2d, 0xb9, 0x17, 0x46, 0x56, + 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, + 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 +}; +static const u8 enc_assoc034[] __initconst = { }; +static const u8 enc_nonce034[] __initconst = { + 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, + 0x37, 0x1a, 0x6f, 0xd2 +}; +static const u8 enc_key034[] __initconst = { + 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, + 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, + 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, + 0x2c, 0x42, 0x4f, 0x03, 0xb6, 0x67, 0x1d, 0x71 +}; + +/* wycheproof - misc */ +static const u8 enc_input035[] __initconst = { + 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, + 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, + 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, + 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb +}; +static const u8 enc_output035[] __initconst = { + 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, + 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, + 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, + 0xd6, 0xf3, 0xd3, 0x26, 0xbc, 0x7b, 0x70, 0x53, + 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, + 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f +}; +static const u8 enc_assoc035[] __initconst = { + 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 +}; +static const u8 enc_nonce035[] __initconst = { + 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, + 0xb8, 0x1a, 0x1f, 0x8b +}; +static const u8 enc_key035[] __initconst = { + 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, + 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, + 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, + 0x37, 0xf1, 0x9e, 0xa4, 0x99, 0x11, 0xce, 0xc4 +}; + +/* wycheproof - misc */ +static const u8 enc_input036[] __initconst = { + 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, + 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, + 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, + 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 +}; +static const u8 enc_output036[] __initconst = { + 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, + 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, + 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, + 0x81, 0xac, 0x43, 0x0b, 0x95, 0x3b, 0x0e, 0x21, + 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, + 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea +}; +static const u8 enc_assoc036[] __initconst = { }; +static const u8 enc_nonce036[] __initconst = { + 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, + 0x2b, 0x7e, 0x6e, 0x6a +}; +static const u8 enc_key036[] __initconst = { + 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, + 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, + 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, + 0xdc, 0x8c, 0x7b, 0xc5, 0x35, 0x46, 0x34, 0xa3 +}; + +/* wycheproof - misc */ +static const u8 enc_input037[] __initconst = { + 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, + 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, + 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, + 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e +}; +static const u8 enc_output037[] __initconst = { + 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, + 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, + 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, + 0xda, 0xf5, 0x07, 0x62, 0x84, 0x6f, 0xf2, 0xf4, + 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, + 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 +}; +static const u8 enc_assoc037[] __initconst = { + 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 +}; +static const u8 enc_nonce037[] __initconst = { + 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, + 0x34, 0x0d, 0xd1, 0xb8 +}; +static const u8 enc_key037[] __initconst = { + 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, + 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, + 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, + 0xe2, 0x5b, 0x78, 0x6a, 0x2d, 0x97, 0xf1, 0x45 +}; + +/* wycheproof - misc */ +static const u8 enc_input038[] __initconst = { + 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, + 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, + 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, + 0x71, 0x34, 0x6b, 0x4d, 0x2c, 0xeb, 0xf8, 0x04, + 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, + 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a +}; +static const u8 enc_output038[] __initconst = { + 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, + 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, + 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, + 0x52, 0xe7, 0x6a, 0x43, 0x2c, 0x65, 0x8d, 0x27, + 0x09, 0x5f, 0x0e, 0xb8, 0xf9, 0x40, 0xc3, 0x24, + 0x98, 0x1e, 0xa9, 0x35, 0xe5, 0x07, 0xf9, 0x8f, + 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, + 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 +}; +static const u8 enc_assoc038[] __initconst = { }; +static const u8 enc_nonce038[] __initconst = { + 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, + 0x0b, 0xef, 0x55, 0xd2 +}; +static const u8 enc_key038[] __initconst = { + 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, + 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, + 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, + 0xa0, 0xa5, 0xe8, 0xe1, 0xaa, 0xe0, 0xf2, 0x64 +}; + +/* wycheproof - misc */ +static const u8 enc_input039[] __initconst = { + 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, + 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, + 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, + 0x5e, 0x7b, 0x0e, 0x79, 0xef, 0x65, 0x36, 0x59, + 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, + 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 +}; +static const u8 enc_output039[] __initconst = { + 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, + 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, + 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, + 0x19, 0x98, 0x6b, 0x3e, 0x31, 0x8c, 0xae, 0x8d, + 0x80, 0x61, 0x98, 0xfb, 0x4c, 0x52, 0x7c, 0xc3, + 0x93, 0x50, 0xeb, 0xdd, 0xea, 0xc5, 0x73, 0xc4, + 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, + 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 +}; +static const u8 enc_assoc039[] __initconst = { + 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 +}; +static const u8 enc_nonce039[] __initconst = { + 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, + 0x31, 0xcd, 0x4d, 0x95 +}; +static const u8 enc_key039[] __initconst = { + 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, + 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, + 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, + 0x2e, 0xb5, 0xbe, 0x10, 0x9f, 0xdc, 0xb2, 0xac +}; + +/* wycheproof - misc */ +static const u8 enc_input040[] __initconst = { + 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, + 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, + 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, + 0x1e, 0x40, 0x79, 0xcb, 0x64, 0xd0, 0xdc, 0x8f, + 0x5e, 0x67, 0x11, 0xcd, 0x49, 0x21, 0xa7, 0x88, + 0x7d, 0xe7, 0x6e, 0x26, 0x78, 0xfd, 0xc6, 0x76, + 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, + 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 +}; +static const u8 enc_output040[] __initconst = { + 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, + 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, + 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, + 0x12, 0xbb, 0x51, 0xbb, 0x18, 0x72, 0xc8, 0xe8, + 0xb8, 0x77, 0x67, 0x8b, 0xec, 0x44, 0x2c, 0xfc, + 0xbb, 0x0f, 0xf4, 0x64, 0xa6, 0x4b, 0x74, 0x33, + 0x2c, 0xf0, 0x72, 0x89, 0x8c, 0x7e, 0x0e, 0xdd, + 0xf6, 0x23, 0x2e, 0xa6, 0xe2, 0x7e, 0xfe, 0x50, + 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, + 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 +}; +static const u8 enc_assoc040[] __initconst = { }; +static const u8 enc_nonce040[] __initconst = { + 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, + 0x23, 0xcc, 0x06, 0x5b +}; +static const u8 enc_key040[] __initconst = { + 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, + 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, + 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, + 0x52, 0x22, 0x6b, 0xaa, 0x1c, 0x6f, 0xb7, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input041[] __initconst = { + 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, + 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, + 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, + 0x3a, 0x25, 0x01, 0x62, 0x93, 0xed, 0xba, 0x44, + 0x3c, 0x58, 0xe0, 0x7e, 0x7b, 0x71, 0x95, 0xec, + 0x5b, 0xd8, 0x45, 0x82, 0xa9, 0xd5, 0x6c, 0x8d, + 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, + 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 +}; +static const u8 enc_output041[] __initconst = { + 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, + 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, + 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, + 0x71, 0xee, 0x8e, 0x73, 0x82, 0x0c, 0x82, 0x97, + 0x49, 0xf1, 0xab, 0xb4, 0x29, 0x43, 0x67, 0x84, + 0x9f, 0xb6, 0xc2, 0xaa, 0x56, 0xbd, 0xa8, 0xa3, + 0x07, 0x8f, 0x72, 0x3d, 0x7c, 0x1c, 0x85, 0x20, + 0x24, 0xb0, 0x17, 0xb5, 0x89, 0x73, 0xfb, 0x1e, + 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, + 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec +}; +static const u8 enc_assoc041[] __initconst = { + 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 +}; +static const u8 enc_nonce041[] __initconst = { + 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, + 0xe4, 0xeb, 0xb1, 0x9c +}; +static const u8 enc_key041[] __initconst = { + 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, + 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, + 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, + 0x72, 0x52, 0x35, 0x19, 0x0e, 0x10, 0xa9, 0x7e +}; + +/* wycheproof - misc */ +static const u8 enc_input042[] __initconst = { + 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, + 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, + 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, + 0x9d, 0x12, 0xd4, 0x5c, 0x22, 0x21, 0x75, 0x64, + 0x12, 0x15, 0x88, 0x03, 0x22, 0x97, 0xef, 0xf5, + 0x67, 0x83, 0x74, 0x2a, 0x5f, 0xc2, 0x2d, 0x74, + 0x10, 0xff, 0xb2, 0x9d, 0x66, 0x09, 0x86, 0x61, + 0xd7, 0x6f, 0x12, 0x6c, 0x3c, 0x27, 0x68, 0x9e, + 0x43, 0xb3, 0x72, 0x67, 0xca, 0xc5, 0xa3, 0xa6, + 0xd3, 0xab, 0x49, 0xe3, 0x91, 0xda, 0x29, 0xcd, + 0x30, 0x54, 0xa5, 0x69, 0x2e, 0x28, 0x07, 0xe4, + 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, + 0x92 +}; +static const u8 enc_output042[] __initconst = { + 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, + 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, + 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, + 0x51, 0x5e, 0x1b, 0xc9, 0xa6, 0x4f, 0x8a, 0xd5, + 0xbe, 0x8f, 0x07, 0x21, 0xbd, 0xe5, 0x06, 0x45, + 0xd0, 0x00, 0x83, 0xc3, 0xa2, 0x63, 0xa3, 0x10, + 0x53, 0xb7, 0x60, 0x24, 0x5f, 0x52, 0xae, 0x28, + 0x66, 0xa5, 0xec, 0x83, 0xb1, 0x9f, 0x61, 0xbe, + 0x1d, 0x30, 0xd5, 0xc5, 0xd9, 0xfe, 0xcc, 0x4c, + 0xbb, 0xe0, 0x8f, 0xd3, 0x85, 0x81, 0x3a, 0x2a, + 0xa3, 0x9a, 0x00, 0xff, 0x9c, 0x10, 0xf7, 0xf2, + 0x37, 0x02, 0xad, 0xd1, 0xe4, 0xb2, 0xff, 0xa3, + 0x1c, 0x41, 0x86, 0x5f, 0xc7, 0x1d, 0xe1, 0x2b, + 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, + 0xb0 +}; +static const u8 enc_assoc042[] __initconst = { }; +static const u8 enc_nonce042[] __initconst = { + 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, + 0xac, 0xde, 0x27, 0x99 +}; +static const u8 enc_key042[] __initconst = { + 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, + 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, + 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, + 0x41, 0x99, 0x5e, 0x78, 0x33, 0x51, 0xf9, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input043[] __initconst = { + 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, + 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, + 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, + 0x16, 0x5d, 0x06, 0xc2, 0xf5, 0x10, 0x0f, 0xf5, + 0xe8, 0xab, 0x28, 0x82, 0xc4, 0x75, 0xaf, 0xcd, + 0x05, 0xcc, 0xd4, 0x9f, 0x2e, 0x7d, 0x8f, 0x55, + 0xef, 0x3a, 0x72, 0xe3, 0xdc, 0x51, 0xd6, 0x85, + 0x2b, 0x8e, 0x6b, 0x9e, 0x7a, 0xec, 0xe5, 0x7b, + 0xe6, 0x55, 0x6b, 0x0b, 0x6d, 0x94, 0x13, 0xe3, + 0x3f, 0xc5, 0xfc, 0x24, 0xa9, 0xa2, 0x05, 0xad, + 0x59, 0x57, 0x4b, 0xb3, 0x9d, 0x94, 0x4a, 0x92, + 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, + 0x76 +}; +static const u8 enc_output043[] __initconst = { + 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, + 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, + 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, + 0x71, 0x10, 0xac, 0x4d, 0x26, 0x20, 0xc2, 0xae, + 0xc2, 0xf8, 0x48, 0xf5, 0x6d, 0xee, 0xb0, 0x37, + 0xa8, 0xdc, 0xed, 0x75, 0xaf, 0xa8, 0xa6, 0xc8, + 0x90, 0xe2, 0xde, 0xe4, 0x2f, 0x95, 0x0b, 0xb3, + 0x3d, 0x9e, 0x24, 0x24, 0xd0, 0x8a, 0x50, 0x5d, + 0x89, 0x95, 0x63, 0x97, 0x3e, 0xd3, 0x88, 0x70, + 0xf3, 0xde, 0x6e, 0xe2, 0xad, 0xc7, 0xfe, 0x07, + 0x2c, 0x36, 0x6c, 0x14, 0xe2, 0xcf, 0x7c, 0xa6, + 0x2f, 0xb3, 0xd3, 0x6b, 0xee, 0x11, 0x68, 0x54, + 0x61, 0xb7, 0x0d, 0x44, 0xef, 0x8c, 0x66, 0xc5, + 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, + 0xf6 +}; +static const u8 enc_assoc043[] __initconst = { + 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 +}; +static const u8 enc_nonce043[] __initconst = { + 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, + 0xe5, 0x22, 0x94, 0x60 +}; +static const u8 enc_key043[] __initconst = { + 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, + 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, + 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, + 0x15, 0xbd, 0xd2, 0xa4, 0xa5, 0x68, 0x2c, 0x4e +}; + +/* wycheproof - misc */ +static const u8 enc_input044[] __initconst = { + 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, + 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 +}; +static const u8 enc_output044[] __initconst = { + 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, + 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, + 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, + 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 +}; +static const u8 enc_assoc044[] __initconst = { + 0x02 +}; +static const u8 enc_nonce044[] __initconst = { + 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, + 0x02, 0xd5, 0x06, 0x56 +}; +static const u8 enc_key044[] __initconst = { + 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, + 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, + 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, + 0x22, 0x77, 0x0f, 0x2c, 0xe2, 0xf3, 0x14, 0x3c +}; + +/* wycheproof - misc */ +static const u8 enc_input045[] __initconst = { + 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, + 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 +}; +static const u8 enc_output045[] __initconst = { + 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, + 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, + 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, + 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 +}; +static const u8 enc_assoc045[] __initconst = { + 0xb6, 0x48 +}; +static const u8 enc_nonce045[] __initconst = { + 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, + 0x5b, 0x3a, 0xa7, 0x13 +}; +static const u8 enc_key045[] __initconst = { + 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, + 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, + 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, + 0x4b, 0x6e, 0xc7, 0x2c, 0x3b, 0xd8, 0xe4, 0xdc +}; + +/* wycheproof - misc */ +static const u8 enc_input046[] __initconst = { + 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, + 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 +}; +static const u8 enc_output046[] __initconst = { + 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, + 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, + 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, + 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 +}; +static const u8 enc_assoc046[] __initconst = { + 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, + 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 +}; +static const u8 enc_nonce046[] __initconst = { + 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, + 0xa4, 0x65, 0x96, 0xdf +}; +static const u8 enc_key046[] __initconst = { + 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, + 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, + 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, + 0x29, 0xfc, 0xb3, 0x66, 0x90, 0xe6, 0x39, 0x4f +}; + +/* wycheproof - misc */ +static const u8 enc_input047[] __initconst = { + 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, + 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 +}; +static const u8 enc_output047[] __initconst = { + 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, + 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, + 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, + 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f +}; +static const u8 enc_assoc047[] __initconst = { + 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, + 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b +}; +static const u8 enc_nonce047[] __initconst = { + 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, + 0x80, 0x92, 0x66, 0xd9 +}; +static const u8 enc_key047[] __initconst = { + 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, + 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, + 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, + 0x7d, 0x4f, 0x63, 0x3f, 0x2c, 0x08, 0xf0, 0x16 +}; + +/* wycheproof - misc */ +static const u8 enc_input048[] __initconst = { + 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, + 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 +}; +static const u8 enc_output048[] __initconst = { + 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, + 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, + 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, + 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 +}; +static const u8 enc_assoc048[] __initconst = { + 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, + 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, + 0x0e +}; +static const u8 enc_nonce048[] __initconst = { + 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, + 0xfc, 0x10, 0x68, 0xc3 +}; +static const u8 enc_key048[] __initconst = { + 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, + 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, + 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, + 0x2e, 0x93, 0xfd, 0xdc, 0x4b, 0xa7, 0x76, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input049[] __initconst = { + 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, + 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 +}; +static const u8 enc_output049[] __initconst = { + 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, + 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, + 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, + 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 +}; +static const u8 enc_assoc049[] __initconst = { + 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, + 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, + 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, + 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda +}; +static const u8 enc_nonce049[] __initconst = { + 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, + 0xad, 0x14, 0xd5, 0x3e +}; +static const u8 enc_key049[] __initconst = { + 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, + 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, + 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, + 0x84, 0xf6, 0xf9, 0x29, 0x03, 0x30, 0xbe, 0xe0 +}; + +/* wycheproof - misc */ +static const u8 enc_input050[] __initconst = { + 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, + 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c +}; +static const u8 enc_output050[] __initconst = { + 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, + 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, + 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, + 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 +}; +static const u8 enc_assoc050[] __initconst = { + 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, + 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, + 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, + 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 +}; +static const u8 enc_nonce050[] __initconst = { + 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, + 0x6a, 0x7b, 0x76, 0x51 +}; +static const u8 enc_key050[] __initconst = { + 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, + 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, + 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, + 0xb0, 0x6d, 0x6c, 0x58, 0x07, 0x20, 0x51, 0x30 +}; + +/* wycheproof - misc */ +static const u8 enc_input051[] __initconst = { + 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, + 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 +}; +static const u8 enc_output051[] __initconst = { + 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, + 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, + 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, + 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 +}; +static const u8 enc_assoc051[] __initconst = { + 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, + 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, + 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, + 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, + 0x73 +}; +static const u8 enc_nonce051[] __initconst = { + 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, + 0xa8, 0xfa, 0x89, 0x49 +}; +static const u8 enc_key051[] __initconst = { + 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, + 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, + 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, + 0x65, 0x1e, 0xc7, 0xe1, 0x6b, 0x4c, 0x11, 0x99 +}; + +/* wycheproof - misc */ +static const u8 enc_input052[] __initconst = { + 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, + 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed +}; +static const u8 enc_output052[] __initconst = { + 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, + 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, + 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, + 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 +}; +static const u8 enc_assoc052[] __initconst = { + 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, + 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, + 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, + 0x84, 0x58, 0x3d, 0xa7, 0x3d, 0x7f, 0xf8, 0x5c, + 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, + 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 +}; +static const u8 enc_nonce052[] __initconst = { + 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, + 0x0e, 0xc3, 0x5d, 0x12 +}; +static const u8 enc_key052[] __initconst = { + 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, + 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, + 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, + 0xb7, 0xf7, 0x97, 0x49, 0xbd, 0x90, 0xaa, 0xf4 +}; + /* wycheproof - misc */ static const u8 enc_input053[] __initconst = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, @@ -2760,6 +3859,126 @@ static const u8 enc_key073[] __initconst = { }; /* wycheproof - checking for int overflows */ +static const u8 enc_input074[] __initconst = { + 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, + 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, + 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, + 0xac, 0xc1, 0x6a, 0x57, 0x26, 0xbe, 0x71, 0x90, + 0xc6, 0x3c, 0x5a, 0x1c, 0x92, 0x65, 0x84, 0xa0, + 0x96, 0x75, 0x68, 0x28, 0xdc, 0xdc, 0x64, 0xac, + 0xdf, 0x96, 0x3d, 0x93, 0x1b, 0xf1, 0xda, 0xe2, + 0x38, 0xf3, 0xf1, 0x57, 0x22, 0x4a, 0xc4, 0xb5, + 0x42, 0xd7, 0x85, 0xb0, 0xdd, 0x84, 0xdb, 0x6b, + 0xe3, 0xbc, 0x5a, 0x36, 0x63, 0xe8, 0x41, 0x49, + 0xff, 0xbe, 0xd0, 0x9e, 0x54, 0xf7, 0x8f, 0x16, + 0xa8, 0x22, 0x3b, 0x24, 0xcb, 0x01, 0x9f, 0x58, + 0xb2, 0x1b, 0x0e, 0x55, 0x1e, 0x7a, 0xa0, 0x73, + 0x27, 0x62, 0x95, 0x51, 0x37, 0x6c, 0xcb, 0xc3, + 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, + 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a +}; +static const u8 enc_output074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, + 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca +}; +static const u8 enc_assoc074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x02, 0x50, 0x6e +}; +static const u8 enc_key074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input075[] __initconst = { + 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, + 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, + 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, + 0x2d, 0x85, 0x9e, 0x3f, 0xf8, 0x22, 0xec, 0xed, + 0x3a, 0x6d, 0xd9, 0xa6, 0x0f, 0x22, 0x95, 0x7f, + 0x7b, 0x7c, 0x85, 0x7e, 0x88, 0x22, 0xeb, 0x9f, + 0xe0, 0xb8, 0xd7, 0x02, 0x21, 0x41, 0xf2, 0xd0, + 0xb4, 0x8f, 0x4b, 0x56, 0x12, 0xd3, 0x22, 0xa8, + 0x8d, 0xd0, 0xfe, 0x0b, 0x4d, 0x91, 0x79, 0x32, + 0x4f, 0x7c, 0x6c, 0x9e, 0x99, 0x0e, 0xfb, 0xd8, + 0x0e, 0x5e, 0xd6, 0x77, 0x58, 0x26, 0x49, 0x8b, + 0x1e, 0xfe, 0x0f, 0x71, 0xa0, 0xf3, 0xec, 0x5b, + 0x29, 0xcb, 0x28, 0xc2, 0x54, 0x0a, 0x7d, 0xcd, + 0x51, 0xb7, 0xda, 0xae, 0xe0, 0xff, 0x4a, 0x7f, + 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, + 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 +}; +static const u8 enc_output075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, + 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 +}; +static const u8 enc_assoc075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x03, 0x18, 0xa5 +}; +static const u8 enc_key075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ static const u8 enc_input076[] __initconst = { 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, 0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02, @@ -3349,6 +4568,286 @@ static const u8 enc_key085[] __initconst = { 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; +/* wycheproof - special case tag */ +static const u8 enc_input086[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output086[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const u8 enc_assoc086[] __initconst = { + 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, + 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 +}; +static const u8 enc_nonce086[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key086[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input087[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, + 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 +}; +static const u8 enc_nonce087[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key087[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input088[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output088[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_assoc088[] __initconst = { + 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, + 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 +}; +static const u8 enc_nonce088[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key088[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input089[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output089[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_assoc089[] __initconst = { + 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, + 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 +}; +static const u8 enc_nonce089[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key089[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input090[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_assoc090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, + 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 +}; +static const u8 enc_nonce090[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key090[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input091[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output091[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc091[] __initconst = { + 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, + 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 +}; +static const u8 enc_nonce091[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key091[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input092[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output092[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc092[] __initconst = { + 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, + 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 +}; +static const u8 enc_nonce092[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key092[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + /* wycheproof - edge case intermediate sums in poly1305 */ static const u8 enc_input093[] __initconst = { 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, @@ -4455,6 +5954,86 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input011), sizeof(enc_assoc011), sizeof(enc_nonce011) }, { enc_input012, enc_output012, enc_assoc012, enc_nonce012, enc_key012, sizeof(enc_input012), sizeof(enc_assoc012), sizeof(enc_nonce012) }, + { enc_input013, enc_output013, enc_assoc013, enc_nonce013, enc_key013, + sizeof(enc_input013), sizeof(enc_assoc013), sizeof(enc_nonce013) }, + { enc_input014, enc_output014, enc_assoc014, enc_nonce014, enc_key014, + sizeof(enc_input014), sizeof(enc_assoc014), sizeof(enc_nonce014) }, + { enc_input015, enc_output015, enc_assoc015, enc_nonce015, enc_key015, + sizeof(enc_input015), sizeof(enc_assoc015), sizeof(enc_nonce015) }, + { enc_input016, enc_output016, enc_assoc016, enc_nonce016, enc_key016, + sizeof(enc_input016), sizeof(enc_assoc016), sizeof(enc_nonce016) }, + { enc_input017, enc_output017, enc_assoc017, enc_nonce017, enc_key017, + sizeof(enc_input017), sizeof(enc_assoc017), sizeof(enc_nonce017) }, + { enc_input018, enc_output018, enc_assoc018, enc_nonce018, enc_key018, + sizeof(enc_input018), sizeof(enc_assoc018), sizeof(enc_nonce018) }, + { enc_input019, enc_output019, enc_assoc019, enc_nonce019, enc_key019, + sizeof(enc_input019), sizeof(enc_assoc019), sizeof(enc_nonce019) }, + { enc_input020, enc_output020, enc_assoc020, enc_nonce020, enc_key020, + sizeof(enc_input020), sizeof(enc_assoc020), sizeof(enc_nonce020) }, + { enc_input021, enc_output021, enc_assoc021, enc_nonce021, enc_key021, + sizeof(enc_input021), sizeof(enc_assoc021), sizeof(enc_nonce021) }, + { enc_input022, enc_output022, enc_assoc022, enc_nonce022, enc_key022, + sizeof(enc_input022), sizeof(enc_assoc022), sizeof(enc_nonce022) }, + { enc_input023, enc_output023, enc_assoc023, enc_nonce023, enc_key023, + sizeof(enc_input023), sizeof(enc_assoc023), sizeof(enc_nonce023) }, + { enc_input024, enc_output024, enc_assoc024, enc_nonce024, enc_key024, + sizeof(enc_input024), sizeof(enc_assoc024), sizeof(enc_nonce024) }, + { enc_input025, enc_output025, enc_assoc025, enc_nonce025, enc_key025, + sizeof(enc_input025), sizeof(enc_assoc025), sizeof(enc_nonce025) }, + { enc_input026, enc_output026, enc_assoc026, enc_nonce026, enc_key026, + sizeof(enc_input026), sizeof(enc_assoc026), sizeof(enc_nonce026) }, + { enc_input027, enc_output027, enc_assoc027, enc_nonce027, enc_key027, + sizeof(enc_input027), sizeof(enc_assoc027), sizeof(enc_nonce027) }, + { enc_input028, enc_output028, enc_assoc028, enc_nonce028, enc_key028, + sizeof(enc_input028), sizeof(enc_assoc028), sizeof(enc_nonce028) }, + { enc_input029, enc_output029, enc_assoc029, enc_nonce029, enc_key029, + sizeof(enc_input029), sizeof(enc_assoc029), sizeof(enc_nonce029) }, + { enc_input030, enc_output030, enc_assoc030, enc_nonce030, enc_key030, + sizeof(enc_input030), sizeof(enc_assoc030), sizeof(enc_nonce030) }, + { enc_input031, enc_output031, enc_assoc031, enc_nonce031, enc_key031, + sizeof(enc_input031), sizeof(enc_assoc031), sizeof(enc_nonce031) }, + { enc_input032, enc_output032, enc_assoc032, enc_nonce032, enc_key032, + sizeof(enc_input032), sizeof(enc_assoc032), sizeof(enc_nonce032) }, + { enc_input033, enc_output033, enc_assoc033, enc_nonce033, enc_key033, + sizeof(enc_input033), sizeof(enc_assoc033), sizeof(enc_nonce033) }, + { enc_input034, enc_output034, enc_assoc034, enc_nonce034, enc_key034, + sizeof(enc_input034), sizeof(enc_assoc034), sizeof(enc_nonce034) }, + { enc_input035, enc_output035, enc_assoc035, enc_nonce035, enc_key035, + sizeof(enc_input035), sizeof(enc_assoc035), sizeof(enc_nonce035) }, + { enc_input036, enc_output036, enc_assoc036, enc_nonce036, enc_key036, + sizeof(enc_input036), sizeof(enc_assoc036), sizeof(enc_nonce036) }, + { enc_input037, enc_output037, enc_assoc037, enc_nonce037, enc_key037, + sizeof(enc_input037), sizeof(enc_assoc037), sizeof(enc_nonce037) }, + { enc_input038, enc_output038, enc_assoc038, enc_nonce038, enc_key038, + sizeof(enc_input038), sizeof(enc_assoc038), sizeof(enc_nonce038) }, + { enc_input039, enc_output039, enc_assoc039, enc_nonce039, enc_key039, + sizeof(enc_input039), sizeof(enc_assoc039), sizeof(enc_nonce039) }, + { enc_input040, enc_output040, enc_assoc040, enc_nonce040, enc_key040, + sizeof(enc_input040), sizeof(enc_assoc040), sizeof(enc_nonce040) }, + { enc_input041, enc_output041, enc_assoc041, enc_nonce041, enc_key041, + sizeof(enc_input041), sizeof(enc_assoc041), sizeof(enc_nonce041) }, + { enc_input042, enc_output042, enc_assoc042, enc_nonce042, enc_key042, + sizeof(enc_input042), sizeof(enc_assoc042), sizeof(enc_nonce042) }, + { enc_input043, enc_output043, enc_assoc043, enc_nonce043, enc_key043, + sizeof(enc_input043), sizeof(enc_assoc043), sizeof(enc_nonce043) }, + { enc_input044, enc_output044, enc_assoc044, enc_nonce044, enc_key044, + sizeof(enc_input044), sizeof(enc_assoc044), sizeof(enc_nonce044) }, + { enc_input045, enc_output045, enc_assoc045, enc_nonce045, enc_key045, + sizeof(enc_input045), sizeof(enc_assoc045), sizeof(enc_nonce045) }, + { enc_input046, enc_output046, enc_assoc046, enc_nonce046, enc_key046, + sizeof(enc_input046), sizeof(enc_assoc046), sizeof(enc_nonce046) }, + { enc_input047, enc_output047, enc_assoc047, enc_nonce047, enc_key047, + sizeof(enc_input047), sizeof(enc_assoc047), sizeof(enc_nonce047) }, + { enc_input048, enc_output048, enc_assoc048, enc_nonce048, enc_key048, + sizeof(enc_input048), sizeof(enc_assoc048), sizeof(enc_nonce048) }, + { enc_input049, enc_output049, enc_assoc049, enc_nonce049, enc_key049, + sizeof(enc_input049), sizeof(enc_assoc049), sizeof(enc_nonce049) }, + { enc_input050, enc_output050, enc_assoc050, enc_nonce050, enc_key050, + sizeof(enc_input050), sizeof(enc_assoc050), sizeof(enc_nonce050) }, + { enc_input051, enc_output051, enc_assoc051, enc_nonce051, enc_key051, + sizeof(enc_input051), sizeof(enc_assoc051), sizeof(enc_nonce051) }, + { enc_input052, enc_output052, enc_assoc052, enc_nonce052, enc_key052, + sizeof(enc_input052), sizeof(enc_assoc052), sizeof(enc_nonce052) }, { enc_input053, enc_output053, enc_assoc053, enc_nonce053, enc_key053, sizeof(enc_input053), sizeof(enc_assoc053), sizeof(enc_nonce053) }, { enc_input054, enc_output054, enc_assoc054, enc_nonce054, enc_key054, @@ -4497,6 +6076,10 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input072), sizeof(enc_assoc072), sizeof(enc_nonce072) }, { enc_input073, enc_output073, enc_assoc073, enc_nonce073, enc_key073, sizeof(enc_input073), sizeof(enc_assoc073), sizeof(enc_nonce073) }, + { enc_input074, enc_output074, enc_assoc074, enc_nonce074, enc_key074, + sizeof(enc_input074), sizeof(enc_assoc074), sizeof(enc_nonce074) }, + { enc_input075, enc_output075, enc_assoc075, enc_nonce075, enc_key075, + sizeof(enc_input075), sizeof(enc_assoc075), sizeof(enc_nonce075) }, { enc_input076, enc_output076, enc_assoc076, enc_nonce076, enc_key076, sizeof(enc_input076), sizeof(enc_assoc076), sizeof(enc_nonce076) }, { enc_input077, enc_output077, enc_assoc077, enc_nonce077, enc_key077, @@ -4517,6 +6100,20 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input084), sizeof(enc_assoc084), sizeof(enc_nonce084) }, { enc_input085, enc_output085, enc_assoc085, enc_nonce085, enc_key085, sizeof(enc_input085), sizeof(enc_assoc085), sizeof(enc_nonce085) }, + { enc_input086, enc_output086, enc_assoc086, enc_nonce086, enc_key086, + sizeof(enc_input086), sizeof(enc_assoc086), sizeof(enc_nonce086) }, + { enc_input087, enc_output087, enc_assoc087, enc_nonce087, enc_key087, + sizeof(enc_input087), sizeof(enc_assoc087), sizeof(enc_nonce087) }, + { enc_input088, enc_output088, enc_assoc088, enc_nonce088, enc_key088, + sizeof(enc_input088), sizeof(enc_assoc088), sizeof(enc_nonce088) }, + { enc_input089, enc_output089, enc_assoc089, enc_nonce089, enc_key089, + sizeof(enc_input089), sizeof(enc_assoc089), sizeof(enc_nonce089) }, + { enc_input090, enc_output090, enc_assoc090, enc_nonce090, enc_key090, + sizeof(enc_input090), sizeof(enc_assoc090), sizeof(enc_nonce090) }, + { enc_input091, enc_output091, enc_assoc091, enc_nonce091, enc_key091, + sizeof(enc_input091), sizeof(enc_assoc091), sizeof(enc_nonce091) }, + { enc_input092, enc_output092, enc_assoc092, enc_nonce092, enc_key092, + sizeof(enc_input092), sizeof(enc_assoc092), sizeof(enc_nonce092) }, { enc_input093, enc_output093, enc_assoc093, enc_nonce093, enc_key093, sizeof(enc_input093), sizeof(enc_assoc093), sizeof(enc_nonce093) }, { enc_input094, enc_output094, enc_assoc094, enc_nonce094, enc_key094, @@ -7224,6 +8821,43 @@ xchacha20poly1305_dec_vectors[] __initconst = { sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } }; +/* This is for the selftests-only, since it is only useful for the purpose of + * testing the underlying primitives and interactions. + */ +static void __init +chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[12], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + const u8 *pad0 = page_address(ZERO_PAGE(0)); + struct poly1305_desc_ctx poly1305_state; + u32 chacha20_state[CHACHA_STATE_WORDS]; + union { + u8 block0[POLY1305_KEY_SIZE]; + __le64 lens[2]; + } b = {{ 0 }}; + u8 bottom_row[16] = { 0 }; + u32 le_key[8]; + int i; + + memcpy(&bottom_row[4], nonce, 12); + for (i = 0; i < 8; ++i) + le_key[i] = get_unaligned_le32(key + sizeof(le_key[i]) * i); + chacha_init(chacha20_state, le_key, bottom_row); + chacha20_crypt(chacha20_state, b.block0, b.block0, sizeof(b.block0)); + poly1305_init(&poly1305_state, b.block0); + poly1305_update(&poly1305_state, ad, ad_len); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf); + chacha20_crypt(chacha20_state, dst, src, src_len); + poly1305_update(&poly1305_state, dst, src_len); + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf); + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); + poly1305_final(&poly1305_state, dst + src_len); +} + static void __init chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, @@ -7233,6 +8867,9 @@ chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, if (nonce_len == 8) chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, get_unaligned_le64(nonce), key); + else if (nonce_len == 12) + chacha20poly1305_encrypt_bignonce(dst, src, src_len, ad, + ad_len, nonce, key); else BUG(); } @@ -7248,14 +8885,14 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result) bool __init chacha20poly1305_selftest(void) { enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; - size_t i; - u8 *computed_output = NULL, *heap_src = NULL; - struct scatterlist sg_src; + size_t i, j, k, total_len; + u8 *computed_output = NULL, *input = NULL; bool success = true, ret; + struct scatterlist sg_src[3]; - heap_src = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - if (!heap_src || !computed_output) { + input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + if (!computed_output || !input) { pr_err("chacha20poly1305 self-test malloc: FAIL\n"); success = false; goto out; @@ -7284,17 +8921,17 @@ bool __init chacha20poly1305_selftest(void) for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { if (chacha20poly1305_enc_vectors[i].nlen != 8) continue; - memcpy(heap_src, chacha20poly1305_enc_vectors[i].input, + memcpy(computed_output, chacha20poly1305_enc_vectors[i].input, chacha20poly1305_enc_vectors[i].ilen); - sg_init_one(&sg_src, heap_src, + sg_init_one(sg_src, computed_output, chacha20poly1305_enc_vectors[i].ilen + POLY1305_DIGEST_SIZE); - chacha20poly1305_encrypt_sg_inplace(&sg_src, + ret = chacha20poly1305_encrypt_sg_inplace(sg_src, chacha20poly1305_enc_vectors[i].ilen, chacha20poly1305_enc_vectors[i].assoc, chacha20poly1305_enc_vectors[i].alen, get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), chacha20poly1305_enc_vectors[i].key); - if (memcmp(heap_src, + if (!ret || memcmp(computed_output, chacha20poly1305_enc_vectors[i].output, chacha20poly1305_enc_vectors[i].ilen + POLY1305_DIGEST_SIZE)) { @@ -7326,11 +8963,11 @@ bool __init chacha20poly1305_selftest(void) } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { - memcpy(heap_src, chacha20poly1305_dec_vectors[i].input, + memcpy(computed_output, chacha20poly1305_dec_vectors[i].input, chacha20poly1305_dec_vectors[i].ilen); - sg_init_one(&sg_src, heap_src, + sg_init_one(sg_src, computed_output, chacha20poly1305_dec_vectors[i].ilen); - ret = chacha20poly1305_decrypt_sg_inplace(&sg_src, + ret = chacha20poly1305_decrypt_sg_inplace(sg_src, chacha20poly1305_dec_vectors[i].ilen, chacha20poly1305_dec_vectors[i].assoc, chacha20poly1305_dec_vectors[i].alen, @@ -7338,7 +8975,7 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].key); if (!decryption_success(ret, chacha20poly1305_dec_vectors[i].failure, - memcmp(heap_src, chacha20poly1305_dec_vectors[i].output, + memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, chacha20poly1305_dec_vectors[i].ilen - POLY1305_DIGEST_SIZE))) { pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", @@ -7365,6 +9002,7 @@ bool __init chacha20poly1305_selftest(void) success = false; } } + for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); ret = xchacha20poly1305_decrypt(computed_output, @@ -7386,8 +9024,59 @@ bool __init chacha20poly1305_selftest(void) } } + for (total_len = POLY1305_DIGEST_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) + && total_len <= 1 << 10; ++total_len) { + for (i = 0; i <= total_len; ++i) { + for (j = i; j <= total_len; ++j) { + k = 0; + sg_init_table(sg_src, 3); + if (i) + sg_set_buf(&sg_src[k++], input, i); + if (j - i) + sg_set_buf(&sg_src[k++], input + i, j - i); + if (total_len - j) + sg_set_buf(&sg_src[k++], input + j, total_len - j); + sg_init_marker(sg_src, k); + memset(computed_output, 0, total_len); + memset(input, 0, total_len); + + if (!chacha20poly1305_encrypt_sg_inplace(sg_src, + total_len - POLY1305_DIGEST_SIZE, NULL, 0, + 0, enc_key001)) + goto chunkfail; + chacha20poly1305_encrypt(computed_output, + computed_output, + total_len - POLY1305_DIGEST_SIZE, NULL, 0, 0, + enc_key001); + if (memcmp(computed_output, input, total_len)) + goto chunkfail; + if (!chacha20poly1305_decrypt(computed_output, + input, total_len, NULL, 0, 0, enc_key001)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_DIGEST_SIZE; ++k) { + if (computed_output[k]) + goto chunkfail; + } + if (!chacha20poly1305_decrypt_sg_inplace(sg_src, + total_len, NULL, 0, 0, enc_key001)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_DIGEST_SIZE; ++k) { + if (input[k]) + goto chunkfail; + } + continue; + + chunkfail: + pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", + total_len, i, j); + success = false; + } + + } + } + out: - kfree(heap_src); kfree(computed_output); + kfree(input); return success; } diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 6d83cafebc69..fa6a9440fc95 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -21,8 +21,6 @@ #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) -bool __init chacha20poly1305_selftest(void); - static void chacha_load_key(u32 *k, const u8 *in) { k[0] = get_unaligned_le32(in); @@ -235,6 +233,9 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, __le64 lens[2]; } b __aligned(16); + if (WARN_ON(src_len > INT_MAX)) + return false; + chacha_load_key(b.k, key); b.iv[0] = 0; @@ -250,9 +251,7 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); } - flags = SG_MITER_TO_SG; - if (!preemptible()) - flags |= SG_MITER_ATOMIC; + flags = SG_MITER_TO_SG | SG_MITER_ATOMIC; sg_miter_start(&miter, src, sg_nents(src), flags); @@ -355,7 +354,7 @@ bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len } EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); -static int __init mod_init(void) +static int __init chacha20poly1305_init(void) { if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && WARN_ON(!chacha20poly1305_selftest())) @@ -363,7 +362,12 @@ static int __init mod_init(void) return 0; } -module_init(mod_init); +static void __exit chacha20poly1305_exit(void) +{ +} + +module_init(chacha20poly1305_init); +module_exit(chacha20poly1305_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); diff --git a/lib/crypto/curve25519-generic.c b/lib/crypto/curve25519-generic.c new file mode 100644 index 000000000000..de7c99172fa2 --- /dev/null +++ b/lib/crypto/curve25519-generic.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + * + * This is an implementation of the Curve25519 ECDH algorithm, using either + * a 32-bit implementation or a 64-bit implementation with 128-bit integers, + * depending on what is supported by the target compiler. + * + * Information: https://cr.yp.to/ecdh.html + */ + +#include <crypto/curve25519.h> +#include <linux/module.h> + +const u8 curve25519_null_point[CURVE25519_KEY_SIZE] __aligned(32) = { 0 }; +const u8 curve25519_base_point[CURVE25519_KEY_SIZE] __aligned(32) = { 9 }; + +EXPORT_SYMBOL(curve25519_null_point); +EXPORT_SYMBOL(curve25519_base_point); +EXPORT_SYMBOL(curve25519_generic); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Curve25519 scalar multiplication"); +MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); diff --git a/lib/crypto/curve25519-selftest.c b/lib/crypto/curve25519-selftest.c new file mode 100644 index 000000000000..c85e85381e78 --- /dev/null +++ b/lib/crypto/curve25519-selftest.c @@ -0,0 +1,1321 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include <crypto/curve25519.h> + +struct curve25519_test_vector { + u8 private[CURVE25519_KEY_SIZE]; + u8 public[CURVE25519_KEY_SIZE]; + u8 result[CURVE25519_KEY_SIZE]; + bool valid; +}; +static const struct curve25519_test_vector curve25519_test_vectors[] __initconst = { + { + .private = { 0x77, 0x07, 0x6d, 0x0a, 0x73, 0x18, 0xa5, 0x7d, + 0x3c, 0x16, 0xc1, 0x72, 0x51, 0xb2, 0x66, 0x45, + 0xdf, 0x4c, 0x2f, 0x87, 0xeb, 0xc0, 0x99, 0x2a, + 0xb1, 0x77, 0xfb, 0xa5, 0x1d, 0xb9, 0x2c, 0x2a }, + .public = { 0xde, 0x9e, 0xdb, 0x7d, 0x7b, 0x7d, 0xc1, 0xb4, + 0xd3, 0x5b, 0x61, 0xc2, 0xec, 0xe4, 0x35, 0x37, + 0x3f, 0x83, 0x43, 0xc8, 0x5b, 0x78, 0x67, 0x4d, + 0xad, 0xfc, 0x7e, 0x14, 0x6f, 0x88, 0x2b, 0x4f }, + .result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1, + 0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25, + 0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33, + 0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 }, + .valid = true + }, + { + .private = { 0x5d, 0xab, 0x08, 0x7e, 0x62, 0x4a, 0x8a, 0x4b, + 0x79, 0xe1, 0x7f, 0x8b, 0x83, 0x80, 0x0e, 0xe6, + 0x6f, 0x3b, 0xb1, 0x29, 0x26, 0x18, 0xb6, 0xfd, + 0x1c, 0x2f, 0x8b, 0x27, 0xff, 0x88, 0xe0, 0xeb }, + .public = { 0x85, 0x20, 0xf0, 0x09, 0x89, 0x30, 0xa7, 0x54, + 0x74, 0x8b, 0x7d, 0xdc, 0xb4, 0x3e, 0xf7, 0x5a, + 0x0d, 0xbf, 0x3a, 0x0d, 0x26, 0x38, 0x1a, 0xf4, + 0xeb, 0xa4, 0xa9, 0x8e, 0xaa, 0x9b, 0x4e, 0x6a }, + .result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1, + 0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25, + 0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33, + 0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 }, + .valid = true + }, + { + .private = { 1 }, + .public = { 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x3c, 0x77, 0x77, 0xca, 0xf9, 0x97, 0xb2, 0x64, + 0x41, 0x60, 0x77, 0x66, 0x5b, 0x4e, 0x22, 0x9d, + 0x0b, 0x95, 0x48, 0xdc, 0x0c, 0xd8, 0x19, 0x98, + 0xdd, 0xcd, 0xc5, 0xc8, 0x53, 0x3c, 0x79, 0x7f }, + .valid = true + }, + { + .private = { 1 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xb3, 0x2d, 0x13, 0x62, 0xc2, 0x48, 0xd6, 0x2f, + 0xe6, 0x26, 0x19, 0xcf, 0xf0, 0x4d, 0xd4, 0x3d, + 0xb7, 0x3f, 0xfc, 0x1b, 0x63, 0x08, 0xed, 0xe3, + 0x0b, 0x78, 0xd8, 0x73, 0x80, 0xf1, 0xe8, 0x34 }, + .valid = true + }, + { + .private = { 0xa5, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, + 0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, + 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18, + 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0xc4 }, + .public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, + 0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, + 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b, + 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c }, + .result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, + 0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, + 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7, + 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 }, + .valid = true + }, + { + .private = { 1, 2, 3, 4 }, + .public = { 0 }, + .result = { 0 }, + .valid = false + }, + { + .private = { 2, 4, 6, 8 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8 }, + .result = { 0 }, + .valid = false + }, + { + .private = { 0xff, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x0a, 0x00, 0xfb, 0x9f }, + .result = { 0x77, 0x52, 0xb6, 0x18, 0xc1, 0x2d, 0x48, 0xd2, + 0xc6, 0x93, 0x46, 0x83, 0x81, 0x7c, 0xc6, 0x57, + 0xf3, 0x31, 0x03, 0x19, 0x49, 0x48, 0x20, 0x05, + 0x42, 0x2b, 0x4e, 0xae, 0x8d, 0x1d, 0x43, 0x23 }, + .valid = true + }, + { + .private = { 0x8e, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x06 }, + .result = { 0x5a, 0xdf, 0xaa, 0x25, 0x86, 0x8e, 0x32, 0x3d, + 0xae, 0x49, 0x62, 0xc1, 0x01, 0x5c, 0xb3, 0x12, + 0xe1, 0xc5, 0xc7, 0x9e, 0x95, 0x3f, 0x03, 0x99, + 0xb0, 0xba, 0x16, 0x22, 0xf3, 0xb6, 0xf7, 0x0c }, + .valid = true + }, + /* wycheproof - normal case */ + { + .private = { 0x48, 0x52, 0x83, 0x4d, 0x9d, 0x6b, 0x77, 0xda, + 0xde, 0xab, 0xaa, 0xf2, 0xe1, 0x1d, 0xca, 0x66, + 0xd1, 0x9f, 0xe7, 0x49, 0x93, 0xa7, 0xbe, 0xc3, + 0x6c, 0x6e, 0x16, 0xa0, 0x98, 0x3f, 0xea, 0xba }, + .public = { 0x9c, 0x64, 0x7d, 0x9a, 0xe5, 0x89, 0xb9, 0xf5, + 0x8f, 0xdc, 0x3c, 0xa4, 0x94, 0x7e, 0xfb, 0xc9, + 0x15, 0xc4, 0xb2, 0xe0, 0x8e, 0x74, 0x4a, 0x0e, + 0xdf, 0x46, 0x9d, 0xac, 0x59, 0xc8, 0xf8, 0x5a }, + .result = { 0x87, 0xb7, 0xf2, 0x12, 0xb6, 0x27, 0xf7, 0xa5, + 0x4c, 0xa5, 0xe0, 0xbc, 0xda, 0xdd, 0xd5, 0x38, + 0x9d, 0x9d, 0xe6, 0x15, 0x6c, 0xdb, 0xcf, 0x8e, + 0xbe, 0x14, 0xff, 0xbc, 0xfb, 0x43, 0x65, 0x51 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x58, 0x8c, 0x06, 0x1a, 0x50, 0x80, 0x4a, 0xc4, + 0x88, 0xad, 0x77, 0x4a, 0xc7, 0x16, 0xc3, 0xf5, + 0xba, 0x71, 0x4b, 0x27, 0x12, 0xe0, 0x48, 0x49, + 0x13, 0x79, 0xa5, 0x00, 0x21, 0x19, 0x98, 0xa8 }, + .public = { 0x63, 0xaa, 0x40, 0xc6, 0xe3, 0x83, 0x46, 0xc5, + 0xca, 0xf2, 0x3a, 0x6d, 0xf0, 0xa5, 0xe6, 0xc8, + 0x08, 0x89, 0xa0, 0x86, 0x47, 0xe5, 0x51, 0xb3, + 0x56, 0x34, 0x49, 0xbe, 0xfc, 0xfc, 0x97, 0x33 }, + .result = { 0xb1, 0xa7, 0x07, 0x51, 0x94, 0x95, 0xff, 0xff, + 0xb2, 0x98, 0xff, 0x94, 0x17, 0x16, 0xb0, 0x6d, + 0xfa, 0xb8, 0x7c, 0xf8, 0xd9, 0x11, 0x23, 0xfe, + 0x2b, 0xe9, 0xa2, 0x33, 0xdd, 0xa2, 0x22, 0x12 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0xb0, 0x5b, 0xfd, 0x32, 0xe5, 0x53, 0x25, 0xd9, + 0xfd, 0x64, 0x8c, 0xb3, 0x02, 0x84, 0x80, 0x39, + 0x00, 0x0b, 0x39, 0x0e, 0x44, 0xd5, 0x21, 0xe5, + 0x8a, 0xab, 0x3b, 0x29, 0xa6, 0x96, 0x0b, 0xa8 }, + .public = { 0x0f, 0x83, 0xc3, 0x6f, 0xde, 0xd9, 0xd3, 0x2f, + 0xad, 0xf4, 0xef, 0xa3, 0xae, 0x93, 0xa9, 0x0b, + 0xb5, 0xcf, 0xa6, 0x68, 0x93, 0xbc, 0x41, 0x2c, + 0x43, 0xfa, 0x72, 0x87, 0xdb, 0xb9, 0x97, 0x79 }, + .result = { 0x67, 0xdd, 0x4a, 0x6e, 0x16, 0x55, 0x33, 0x53, + 0x4c, 0x0e, 0x3f, 0x17, 0x2e, 0x4a, 0xb8, 0x57, + 0x6b, 0xca, 0x92, 0x3a, 0x5f, 0x07, 0xb2, 0xc0, + 0x69, 0xb4, 0xc3, 0x10, 0xff, 0x2e, 0x93, 0x5b }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x70, 0xe3, 0x4b, 0xcb, 0xe1, 0xf4, 0x7f, 0xbc, + 0x0f, 0xdd, 0xfd, 0x7c, 0x1e, 0x1a, 0xa5, 0x3d, + 0x57, 0xbf, 0xe0, 0xf6, 0x6d, 0x24, 0x30, 0x67, + 0xb4, 0x24, 0xbb, 0x62, 0x10, 0xbe, 0xd1, 0x9c }, + .public = { 0x0b, 0x82, 0x11, 0xa2, 0xb6, 0x04, 0x90, 0x97, + 0xf6, 0x87, 0x1c, 0x6c, 0x05, 0x2d, 0x3c, 0x5f, + 0xc1, 0xba, 0x17, 0xda, 0x9e, 0x32, 0xae, 0x45, + 0x84, 0x03, 0xb0, 0x5b, 0xb2, 0x83, 0x09, 0x2a }, + .result = { 0x4a, 0x06, 0x38, 0xcf, 0xaa, 0x9e, 0xf1, 0x93, + 0x3b, 0x47, 0xf8, 0x93, 0x92, 0x96, 0xa6, 0xb2, + 0x5b, 0xe5, 0x41, 0xef, 0x7f, 0x70, 0xe8, 0x44, + 0xc0, 0xbc, 0xc0, 0x0b, 0x13, 0x4d, 0xe6, 0x4a }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x68, 0xc1, 0xf3, 0xa6, 0x53, 0xa4, 0xcd, 0xb1, + 0xd3, 0x7b, 0xba, 0x94, 0x73, 0x8f, 0x8b, 0x95, + 0x7a, 0x57, 0xbe, 0xb2, 0x4d, 0x64, 0x6e, 0x99, + 0x4d, 0xc2, 0x9a, 0x27, 0x6a, 0xad, 0x45, 0x8d }, + .public = { 0x34, 0x3a, 0xc2, 0x0a, 0x3b, 0x9c, 0x6a, 0x27, + 0xb1, 0x00, 0x81, 0x76, 0x50, 0x9a, 0xd3, 0x07, + 0x35, 0x85, 0x6e, 0xc1, 0xc8, 0xd8, 0xfc, 0xae, + 0x13, 0x91, 0x2d, 0x08, 0xd1, 0x52, 0xf4, 0x6c }, + .result = { 0x39, 0x94, 0x91, 0xfc, 0xe8, 0xdf, 0xab, 0x73, + 0xb4, 0xf9, 0xf6, 0x11, 0xde, 0x8e, 0xa0, 0xb2, + 0x7b, 0x28, 0xf8, 0x59, 0x94, 0x25, 0x0b, 0x0f, + 0x47, 0x5d, 0x58, 0x5d, 0x04, 0x2a, 0xc2, 0x07 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0xd8, 0x77, 0xb2, 0x6d, 0x06, 0xdf, 0xf9, 0xd9, + 0xf7, 0xfd, 0x4c, 0x5b, 0x37, 0x69, 0xf8, 0xcd, + 0xd5, 0xb3, 0x05, 0x16, 0xa5, 0xab, 0x80, 0x6b, + 0xe3, 0x24, 0xff, 0x3e, 0xb6, 0x9e, 0xa0, 0xb2 }, + .public = { 0xfa, 0x69, 0x5f, 0xc7, 0xbe, 0x8d, 0x1b, 0xe5, + 0xbf, 0x70, 0x48, 0x98, 0xf3, 0x88, 0xc4, 0x52, + 0xba, 0xfd, 0xd3, 0xb8, 0xea, 0xe8, 0x05, 0xf8, + 0x68, 0x1a, 0x8d, 0x15, 0xc2, 0xd4, 0xe1, 0x42 }, + .result = { 0x2c, 0x4f, 0xe1, 0x1d, 0x49, 0x0a, 0x53, 0x86, + 0x17, 0x76, 0xb1, 0x3b, 0x43, 0x54, 0xab, 0xd4, + 0xcf, 0x5a, 0x97, 0x69, 0x9d, 0xb6, 0xe6, 0xc6, + 0x8c, 0x16, 0x26, 0xd0, 0x76, 0x62, 0xf7, 0x58 }, + .valid = true + }, + /* wycheproof - public key = 0 */ + { + .private = { 0x20, 0x74, 0x94, 0x03, 0x8f, 0x2b, 0xb8, 0x11, + 0xd4, 0x78, 0x05, 0xbc, 0xdf, 0x04, 0xa2, 0xac, + 0x58, 0x5a, 0xda, 0x7f, 0x2f, 0x23, 0x38, 0x9b, + 0xfd, 0x46, 0x58, 0xf9, 0xdd, 0xd4, 0xde, 0xbc }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key = 1 */ + { + .private = { 0x20, 0x2e, 0x89, 0x72, 0xb6, 0x1c, 0x7e, 0x61, + 0x93, 0x0e, 0xb9, 0x45, 0x0b, 0x50, 0x70, 0xea, + 0xe1, 0xc6, 0x70, 0x47, 0x56, 0x85, 0x54, 0x1f, + 0x04, 0x76, 0x21, 0x7e, 0x48, 0x18, 0xcf, 0xab }, + .public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x38, 0xdd, 0xe9, 0xf3, 0xe7, 0xb7, 0x99, 0x04, + 0x5f, 0x9a, 0xc3, 0x79, 0x3d, 0x4a, 0x92, 0x77, + 0xda, 0xde, 0xad, 0xc4, 0x1b, 0xec, 0x02, 0x90, + 0xf8, 0x1f, 0x74, 0x4f, 0x73, 0x77, 0x5f, 0x84 }, + .public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x9a, 0x2c, 0xfe, 0x84, 0xff, 0x9c, 0x4a, 0x97, + 0x39, 0x62, 0x5c, 0xae, 0x4a, 0x3b, 0x82, 0xa9, + 0x06, 0x87, 0x7a, 0x44, 0x19, 0x46, 0xf8, 0xd7, + 0xb3, 0xd7, 0x95, 0xfe, 0x8f, 0x5d, 0x16, 0x39 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x98, 0x57, 0xa9, 0x14, 0xe3, 0xc2, 0x90, 0x36, + 0xfd, 0x9a, 0x44, 0x2b, 0xa5, 0x26, 0xb5, 0xcd, + 0xcd, 0xf2, 0x82, 0x16, 0x15, 0x3e, 0x63, 0x6c, + 0x10, 0x67, 0x7a, 0xca, 0xb6, 0xbd, 0x6a, 0xa5 }, + .public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x4d, 0xa4, 0xe0, 0xaa, 0x07, 0x2c, 0x23, 0x2e, + 0xe2, 0xf0, 0xfa, 0x4e, 0x51, 0x9a, 0xe5, 0x0b, + 0x52, 0xc1, 0xed, 0xd0, 0x8a, 0x53, 0x4d, 0x4e, + 0xf3, 0x46, 0xc2, 0xe1, 0x06, 0xd2, 0x1d, 0x60 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x48, 0xe2, 0x13, 0x0d, 0x72, 0x33, 0x05, 0xed, + 0x05, 0xe6, 0xe5, 0x89, 0x4d, 0x39, 0x8a, 0x5e, + 0x33, 0x36, 0x7a, 0x8c, 0x6a, 0xac, 0x8f, 0xcd, + 0xf0, 0xa8, 0x8e, 0x4b, 0x42, 0x82, 0x0d, 0xb7 }, + .public = { 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0xf8, 0xff, + 0xff, 0x1f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0x07, 0x00, + 0x00, 0xf0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00 }, + .result = { 0x9e, 0xd1, 0x0c, 0x53, 0x74, 0x7f, 0x64, 0x7f, + 0x82, 0xf4, 0x51, 0x25, 0xd3, 0xde, 0x15, 0xa1, + 0xe6, 0xb8, 0x24, 0x49, 0x6a, 0xb4, 0x04, 0x10, + 0xff, 0xcc, 0x3c, 0xfe, 0x95, 0x76, 0x0f, 0x3b }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x28, 0xf4, 0x10, 0x11, 0x69, 0x18, 0x51, 0xb3, + 0xa6, 0x2b, 0x64, 0x15, 0x53, 0xb3, 0x0d, 0x0d, + 0xfd, 0xdc, 0xb8, 0xff, 0xfc, 0xf5, 0x37, 0x00, + 0xa7, 0xbe, 0x2f, 0x6a, 0x87, 0x2e, 0x9f, 0xb0 }, + .public = { 0x00, 0x00, 0x00, 0xfc, 0xff, 0xff, 0x07, 0x00, + 0x00, 0xe0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0xf8, 0xff, + 0xff, 0x0f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0x7f }, + .result = { 0xcf, 0x72, 0xb4, 0xaa, 0x6a, 0xa1, 0xc9, 0xf8, + 0x94, 0xf4, 0x16, 0x5b, 0x86, 0x10, 0x9a, 0xa4, + 0x68, 0x51, 0x76, 0x48, 0xe1, 0xf0, 0xcc, 0x70, + 0xe1, 0xab, 0x08, 0x46, 0x01, 0x76, 0x50, 0x6b }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x18, 0xa9, 0x3b, 0x64, 0x99, 0xb9, 0xf6, 0xb3, + 0x22, 0x5c, 0xa0, 0x2f, 0xef, 0x41, 0x0e, 0x0a, + 0xde, 0xc2, 0x35, 0x32, 0x32, 0x1d, 0x2d, 0x8e, + 0xf1, 0xa6, 0xd6, 0x02, 0xa8, 0xc6, 0x5b, 0x83 }, + .public = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x5d, 0x50, 0xb6, 0x28, 0x36, 0xbb, 0x69, 0x57, + 0x94, 0x10, 0x38, 0x6c, 0xf7, 0xbb, 0x81, 0x1c, + 0x14, 0xbf, 0x85, 0xb1, 0xc7, 0xb1, 0x7e, 0x59, + 0x24, 0xc7, 0xff, 0xea, 0x91, 0xef, 0x9e, 0x12 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0xc0, 0x1d, 0x13, 0x05, 0xa1, 0x33, 0x8a, 0x1f, + 0xca, 0xc2, 0xba, 0x7e, 0x2e, 0x03, 0x2b, 0x42, + 0x7e, 0x0b, 0x04, 0x90, 0x31, 0x65, 0xac, 0xa9, + 0x57, 0xd8, 0xd0, 0x55, 0x3d, 0x87, 0x17, 0xb0 }, + .public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x19, 0x23, 0x0e, 0xb1, 0x48, 0xd5, 0xd6, 0x7c, + 0x3c, 0x22, 0xab, 0x1d, 0xae, 0xff, 0x80, 0xa5, + 0x7e, 0xae, 0x42, 0x65, 0xce, 0x28, 0x72, 0x65, + 0x7b, 0x2c, 0x80, 0x99, 0xfc, 0x69, 0x8e, 0x50 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x38, 0x6f, 0x7f, 0x16, 0xc5, 0x07, 0x31, 0xd6, + 0x4f, 0x82, 0xe6, 0xa1, 0x70, 0xb1, 0x42, 0xa4, + 0xe3, 0x4f, 0x31, 0xfd, 0x77, 0x68, 0xfc, 0xb8, + 0x90, 0x29, 0x25, 0xe7, 0xd1, 0xe2, 0x1a, 0xbe }, + .public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x0f, 0xca, 0xb5, 0xd8, 0x42, 0xa0, 0x78, 0xd7, + 0xa7, 0x1f, 0xc5, 0x9b, 0x57, 0xbf, 0xb4, 0xca, + 0x0b, 0xe6, 0x87, 0x3b, 0x49, 0xdc, 0xdb, 0x9f, + 0x44, 0xe1, 0x4a, 0xe8, 0xfb, 0xdf, 0xa5, 0x42 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xe0, 0x23, 0xa2, 0x89, 0xbd, 0x5e, 0x90, 0xfa, + 0x28, 0x04, 0xdd, 0xc0, 0x19, 0xa0, 0x5e, 0xf3, + 0xe7, 0x9d, 0x43, 0x4b, 0xb6, 0xea, 0x2f, 0x52, + 0x2e, 0xcb, 0x64, 0x3a, 0x75, 0x29, 0x6e, 0x95 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x54, 0xce, 0x8f, 0x22, 0x75, 0xc0, 0x77, 0xe3, + 0xb1, 0x30, 0x6a, 0x39, 0x39, 0xc5, 0xe0, 0x3e, + 0xef, 0x6b, 0xbb, 0x88, 0x06, 0x05, 0x44, 0x75, + 0x8d, 0x9f, 0xef, 0x59, 0xb0, 0xbc, 0x3e, 0x4f }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x68, 0xf0, 0x10, 0xd6, 0x2e, 0xe8, 0xd9, 0x26, + 0x05, 0x3a, 0x36, 0x1c, 0x3a, 0x75, 0xc6, 0xea, + 0x4e, 0xbd, 0xc8, 0x60, 0x6a, 0xb2, 0x85, 0x00, + 0x3a, 0x6f, 0x8f, 0x40, 0x76, 0xb0, 0x1e, 0x83 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 }, + .result = { 0xf1, 0x36, 0x77, 0x5c, 0x5b, 0xeb, 0x0a, 0xf8, + 0x11, 0x0a, 0xf1, 0x0b, 0x20, 0x37, 0x23, 0x32, + 0x04, 0x3c, 0xab, 0x75, 0x24, 0x19, 0x67, 0x87, + 0x75, 0xa2, 0x23, 0xdf, 0x57, 0xc9, 0xd3, 0x0d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x58, 0xeb, 0xcb, 0x35, 0xb0, 0xf8, 0x84, 0x5c, + 0xaf, 0x1e, 0xc6, 0x30, 0xf9, 0x65, 0x76, 0xb6, + 0x2c, 0x4b, 0x7b, 0x6c, 0x36, 0xb2, 0x9d, 0xeb, + 0x2c, 0xb0, 0x08, 0x46, 0x51, 0x75, 0x5c, 0x96 }, + .public = { 0xff, 0xff, 0xff, 0xfb, 0xff, 0xff, 0xfb, 0xff, + 0xff, 0xdf, 0xff, 0xff, 0xdf, 0xff, 0xff, 0xff, + 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xf7, 0xff, + 0xff, 0xf7, 0xff, 0xff, 0xbf, 0xff, 0xff, 0x3f }, + .result = { 0xbf, 0x9a, 0xff, 0xd0, 0x6b, 0x84, 0x40, 0x85, + 0x58, 0x64, 0x60, 0x96, 0x2e, 0xf2, 0x14, 0x6f, + 0xf3, 0xd4, 0x53, 0x3d, 0x94, 0x44, 0xaa, 0xb0, + 0x06, 0xeb, 0x88, 0xcc, 0x30, 0x54, 0x40, 0x7d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x18, 0x8c, 0x4b, 0xc5, 0xb9, 0xc4, 0x4b, 0x38, + 0xbb, 0x65, 0x8b, 0x9b, 0x2a, 0xe8, 0x2d, 0x5b, + 0x01, 0x01, 0x5e, 0x09, 0x31, 0x84, 0xb1, 0x7c, + 0xb7, 0x86, 0x35, 0x03, 0xa7, 0x83, 0xe1, 0xbb }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .result = { 0xd4, 0x80, 0xde, 0x04, 0xf6, 0x99, 0xcb, 0x3b, + 0xe0, 0x68, 0x4a, 0x9c, 0xc2, 0xe3, 0x12, 0x81, + 0xea, 0x0b, 0xc5, 0xa9, 0xdc, 0xc1, 0x57, 0xd3, + 0xd2, 0x01, 0x58, 0xd4, 0x6c, 0xa5, 0x24, 0x6d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xe0, 0x6c, 0x11, 0xbb, 0x2e, 0x13, 0xce, 0x3d, + 0xc7, 0x67, 0x3f, 0x67, 0xf5, 0x48, 0x22, 0x42, + 0x90, 0x94, 0x23, 0xa9, 0xae, 0x95, 0xee, 0x98, + 0x6a, 0x98, 0x8d, 0x98, 0xfa, 0xee, 0x23, 0xa2 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f }, + .result = { 0x4c, 0x44, 0x01, 0xcc, 0xe6, 0xb5, 0x1e, 0x4c, + 0xb1, 0x8f, 0x27, 0x90, 0x24, 0x6c, 0x9b, 0xf9, + 0x14, 0xdb, 0x66, 0x77, 0x50, 0xa1, 0xcb, 0x89, + 0x06, 0x90, 0x92, 0xaf, 0x07, 0x29, 0x22, 0x76 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xc0, 0x65, 0x8c, 0x46, 0xdd, 0xe1, 0x81, 0x29, + 0x29, 0x38, 0x77, 0x53, 0x5b, 0x11, 0x62, 0xb6, + 0xf9, 0xf5, 0x41, 0x4a, 0x23, 0xcf, 0x4d, 0x2c, + 0xbc, 0x14, 0x0a, 0x4d, 0x99, 0xda, 0x2b, 0x8f }, + .public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x57, 0x8b, 0xa8, 0xcc, 0x2d, 0xbd, 0xc5, 0x75, + 0xaf, 0xcf, 0x9d, 0xf2, 0xb3, 0xee, 0x61, 0x89, + 0xf5, 0x33, 0x7d, 0x68, 0x54, 0xc7, 0x9b, 0x4c, + 0xe1, 0x65, 0xea, 0x12, 0x29, 0x3b, 0x3a, 0x0f }, + .valid = true + }, + /* wycheproof - public key with low order */ + { + .private = { 0x10, 0x25, 0x5c, 0x92, 0x30, 0xa9, 0x7a, 0x30, + 0xa4, 0x58, 0xca, 0x28, 0x4a, 0x62, 0x96, 0x69, + 0x29, 0x3a, 0x31, 0x89, 0x0c, 0xda, 0x9d, 0x14, + 0x7f, 0xeb, 0xc7, 0xd1, 0xe2, 0x2d, 0x6b, 0xb1 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x78, 0xf1, 0xe8, 0xed, 0xf1, 0x44, 0x81, 0xb3, + 0x89, 0x44, 0x8d, 0xac, 0x8f, 0x59, 0xc7, 0x0b, + 0x03, 0x8e, 0x7c, 0xf9, 0x2e, 0xf2, 0xc7, 0xef, + 0xf5, 0x7a, 0x72, 0x46, 0x6e, 0x11, 0x52, 0x96 }, + .public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, + 0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b, + 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86, + 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0x57 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xa0, 0xa0, 0x5a, 0x3e, 0x8f, 0x9f, 0x44, 0x20, + 0x4d, 0x5f, 0x80, 0x59, 0xa9, 0x4a, 0xc7, 0xdf, + 0xc3, 0x9a, 0x49, 0xac, 0x01, 0x6d, 0xd7, 0x43, + 0xdb, 0xfa, 0x43, 0xc5, 0xd6, 0x71, 0xfd, 0x88 }, + .public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xd0, 0xdb, 0xb3, 0xed, 0x19, 0x06, 0x66, 0x3f, + 0x15, 0x42, 0x0a, 0xf3, 0x1f, 0x4e, 0xaf, 0x65, + 0x09, 0xd9, 0xa9, 0x94, 0x97, 0x23, 0x50, 0x06, + 0x05, 0xad, 0x7c, 0x1c, 0x6e, 0x74, 0x50, 0xa9 }, + .public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xc0, 0xb1, 0xd0, 0xeb, 0x22, 0xb2, 0x44, 0xfe, + 0x32, 0x91, 0x14, 0x00, 0x72, 0xcd, 0xd9, 0xd9, + 0x89, 0xb5, 0xf0, 0xec, 0xd9, 0x6c, 0x10, 0x0f, + 0xeb, 0x5b, 0xca, 0x24, 0x1c, 0x1d, 0x9f, 0x8f }, + .public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x48, 0x0b, 0xf4, 0x5f, 0x59, 0x49, 0x42, 0xa8, + 0xbc, 0x0f, 0x33, 0x53, 0xc6, 0xe8, 0xb8, 0x85, + 0x3d, 0x77, 0xf3, 0x51, 0xf1, 0xc2, 0xca, 0x6c, + 0x2d, 0x1a, 0xbf, 0x8a, 0x00, 0xb4, 0x22, 0x9c }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x30, 0xf9, 0x93, 0xfc, 0xf8, 0x51, 0x4f, 0xc8, + 0x9b, 0xd8, 0xdb, 0x14, 0xcd, 0x43, 0xba, 0x0d, + 0x4b, 0x25, 0x30, 0xe7, 0x3c, 0x42, 0x76, 0xa0, + 0x5e, 0x1b, 0x14, 0x5d, 0x42, 0x0c, 0xed, 0xb4 }, + .public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xc0, 0x49, 0x74, 0xb7, 0x58, 0x38, 0x0e, 0x2a, + 0x5b, 0x5d, 0xf6, 0xeb, 0x09, 0xbb, 0x2f, 0x6b, + 0x34, 0x34, 0xf9, 0x82, 0x72, 0x2a, 0x8e, 0x67, + 0x6d, 0x3d, 0xa2, 0x51, 0xd1, 0xb3, 0xde, 0x83 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x50, 0x2a, 0x31, 0x37, 0x3d, 0xb3, 0x24, 0x46, + 0x84, 0x2f, 0xe5, 0xad, 0xd3, 0xe0, 0x24, 0x02, + 0x2e, 0xa5, 0x4f, 0x27, 0x41, 0x82, 0xaf, 0xc3, + 0xd9, 0xf1, 0xbb, 0x3d, 0x39, 0x53, 0x4e, 0xb5 }, + .public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, + 0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b, + 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86, + 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0xd7 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x90, 0xfa, 0x64, 0x17, 0xb0, 0xe3, 0x70, 0x30, + 0xfd, 0x6e, 0x43, 0xef, 0xf2, 0xab, 0xae, 0xf1, + 0x4c, 0x67, 0x93, 0x11, 0x7a, 0x03, 0x9c, 0xf6, + 0x21, 0x31, 0x8b, 0xa9, 0x0f, 0x4e, 0x98, 0xbe }, + .public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x78, 0xad, 0x3f, 0x26, 0x02, 0x7f, 0x1c, 0x9f, + 0xdd, 0x97, 0x5a, 0x16, 0x13, 0xb9, 0x47, 0x77, + 0x9b, 0xad, 0x2c, 0xf2, 0xb7, 0x41, 0xad, 0xe0, + 0x18, 0x40, 0x88, 0x5a, 0x30, 0xbb, 0x97, 0x9c }, + .public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x98, 0xe2, 0x3d, 0xe7, 0xb1, 0xe0, 0x92, 0x6e, + 0xd9, 0xc8, 0x7e, 0x7b, 0x14, 0xba, 0xf5, 0x5f, + 0x49, 0x7a, 0x1d, 0x70, 0x96, 0xf9, 0x39, 0x77, + 0x68, 0x0e, 0x44, 0xdc, 0x1c, 0x7b, 0x7b, 0x8b }, + .public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf0, 0x1e, 0x48, 0xda, 0xfa, 0xc9, 0xd7, 0xbc, + 0xf5, 0x89, 0xcb, 0xc3, 0x82, 0xc8, 0x78, 0xd1, + 0x8b, 0xda, 0x35, 0x50, 0x58, 0x9f, 0xfb, 0x5d, + 0x50, 0xb5, 0x23, 0xbe, 0xbe, 0x32, 0x9d, 0xae }, + .public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0xbd, 0x36, 0xa0, 0x79, 0x0e, 0xb8, 0x83, 0x09, + 0x8c, 0x98, 0x8b, 0x21, 0x78, 0x67, 0x73, 0xde, + 0x0b, 0x3a, 0x4d, 0xf1, 0x62, 0x28, 0x2c, 0xf1, + 0x10, 0xde, 0x18, 0xdd, 0x48, 0x4c, 0xe7, 0x4b }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x28, 0x87, 0x96, 0xbc, 0x5a, 0xff, 0x4b, 0x81, + 0xa3, 0x75, 0x01, 0x75, 0x7b, 0xc0, 0x75, 0x3a, + 0x3c, 0x21, 0x96, 0x47, 0x90, 0xd3, 0x86, 0x99, + 0x30, 0x8d, 0xeb, 0xc1, 0x7a, 0x6e, 0xaf, 0x8d }, + .public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0xb4, 0xe0, 0xdd, 0x76, 0xda, 0x7b, 0x07, 0x17, + 0x28, 0xb6, 0x1f, 0x85, 0x67, 0x71, 0xaa, 0x35, + 0x6e, 0x57, 0xed, 0xa7, 0x8a, 0x5b, 0x16, 0x55, + 0xcc, 0x38, 0x20, 0xfb, 0x5f, 0x85, 0x4c, 0x5c }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x98, 0xdf, 0x84, 0x5f, 0x66, 0x51, 0xbf, 0x11, + 0x38, 0x22, 0x1f, 0x11, 0x90, 0x41, 0xf7, 0x2b, + 0x6d, 0xbc, 0x3c, 0x4a, 0xce, 0x71, 0x43, 0xd9, + 0x9f, 0xd5, 0x5a, 0xd8, 0x67, 0x48, 0x0d, 0xa8 }, + .public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x6f, 0xdf, 0x6c, 0x37, 0x61, 0x1d, 0xbd, 0x53, + 0x04, 0xdc, 0x0f, 0x2e, 0xb7, 0xc9, 0x51, 0x7e, + 0xb3, 0xc5, 0x0e, 0x12, 0xfd, 0x05, 0x0a, 0xc6, + 0xde, 0xc2, 0x70, 0x71, 0xd4, 0xbf, 0xc0, 0x34 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf0, 0x94, 0x98, 0xe4, 0x6f, 0x02, 0xf8, 0x78, + 0x82, 0x9e, 0x78, 0xb8, 0x03, 0xd3, 0x16, 0xa2, + 0xed, 0x69, 0x5d, 0x04, 0x98, 0xa0, 0x8a, 0xbd, + 0xf8, 0x27, 0x69, 0x30, 0xe2, 0x4e, 0xdc, 0xb0 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x4c, 0x8f, 0xc4, 0xb1, 0xc6, 0xab, 0x88, 0xfb, + 0x21, 0xf1, 0x8f, 0x6d, 0x4c, 0x81, 0x02, 0x40, + 0xd4, 0xe9, 0x46, 0x51, 0xba, 0x44, 0xf7, 0xa2, + 0xc8, 0x63, 0xce, 0xc7, 0xdc, 0x56, 0x60, 0x2d }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x18, 0x13, 0xc1, 0x0a, 0x5c, 0x7f, 0x21, 0xf9, + 0x6e, 0x17, 0xf2, 0x88, 0xc0, 0xcc, 0x37, 0x60, + 0x7c, 0x04, 0xc5, 0xf5, 0xae, 0xa2, 0xdb, 0x13, + 0x4f, 0x9e, 0x2f, 0xfc, 0x66, 0xbd, 0x9d, 0xb8 }, + .public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x1c, 0xd0, 0xb2, 0x82, 0x67, 0xdc, 0x54, 0x1c, + 0x64, 0x2d, 0x6d, 0x7d, 0xca, 0x44, 0xa8, 0xb3, + 0x8a, 0x63, 0x73, 0x6e, 0xef, 0x5c, 0x4e, 0x65, + 0x01, 0xff, 0xbb, 0xb1, 0x78, 0x0c, 0x03, 0x3c }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x78, 0x57, 0xfb, 0x80, 0x86, 0x53, 0x64, 0x5a, + 0x0b, 0xeb, 0x13, 0x8a, 0x64, 0xf5, 0xf4, 0xd7, + 0x33, 0xa4, 0x5e, 0xa8, 0x4c, 0x3c, 0xda, 0x11, + 0xa9, 0xc0, 0x6f, 0x7e, 0x71, 0x39, 0x14, 0x9e }, + .public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x87, 0x55, 0xbe, 0x01, 0xc6, 0x0a, 0x7e, 0x82, + 0x5c, 0xff, 0x3e, 0x0e, 0x78, 0xcb, 0x3a, 0xa4, + 0x33, 0x38, 0x61, 0x51, 0x6a, 0xa5, 0x9b, 0x1c, + 0x51, 0xa8, 0xb2, 0xa5, 0x43, 0xdf, 0xa8, 0x22 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xe0, 0x3a, 0xa8, 0x42, 0xe2, 0xab, 0xc5, 0x6e, + 0x81, 0xe8, 0x7b, 0x8b, 0x9f, 0x41, 0x7b, 0x2a, + 0x1e, 0x59, 0x13, 0xc7, 0x23, 0xee, 0xd2, 0x8d, + 0x75, 0x2f, 0x8d, 0x47, 0xa5, 0x9f, 0x49, 0x8f }, + .public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x54, 0xc9, 0xa1, 0xed, 0x95, 0xe5, 0x46, 0xd2, + 0x78, 0x22, 0xa3, 0x60, 0x93, 0x1d, 0xda, 0x60, + 0xa1, 0xdf, 0x04, 0x9d, 0xa6, 0xf9, 0x04, 0x25, + 0x3c, 0x06, 0x12, 0xbb, 0xdc, 0x08, 0x74, 0x76 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf8, 0xf7, 0x07, 0xb7, 0x99, 0x9b, 0x18, 0xcb, + 0x0d, 0x6b, 0x96, 0x12, 0x4f, 0x20, 0x45, 0x97, + 0x2c, 0xa2, 0x74, 0xbf, 0xc1, 0x54, 0xad, 0x0c, + 0x87, 0x03, 0x8c, 0x24, 0xc6, 0xd0, 0xd4, 0xb2 }, + .public = { 0xda, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xcc, 0x1f, 0x40, 0xd7, 0x43, 0xcd, 0xc2, 0x23, + 0x0e, 0x10, 0x43, 0xda, 0xba, 0x8b, 0x75, 0xe8, + 0x10, 0xf1, 0xfb, 0xab, 0x7f, 0x25, 0x52, 0x69, + 0xbd, 0x9e, 0xbb, 0x29, 0xe6, 0xbf, 0x49, 0x4f }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xa0, 0x34, 0xf6, 0x84, 0xfa, 0x63, 0x1e, 0x1a, + 0x34, 0x81, 0x18, 0xc1, 0xce, 0x4c, 0x98, 0x23, + 0x1f, 0x2d, 0x9e, 0xec, 0x9b, 0xa5, 0x36, 0x5b, + 0x4a, 0x05, 0xd6, 0x9a, 0x78, 0x5b, 0x07, 0x96 }, + .public = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x54, 0x99, 0x8e, 0xe4, 0x3a, 0x5b, 0x00, 0x7b, + 0xf4, 0x99, 0xf0, 0x78, 0xe7, 0x36, 0x52, 0x44, + 0x00, 0xa8, 0xb5, 0xc7, 0xe9, 0xb9, 0xb4, 0x37, + 0x71, 0x74, 0x8c, 0x7c, 0xdf, 0x88, 0x04, 0x12 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x30, 0xb6, 0xc6, 0xa0, 0xf2, 0xff, 0xa6, 0x80, + 0x76, 0x8f, 0x99, 0x2b, 0xa8, 0x9e, 0x15, 0x2d, + 0x5b, 0xc9, 0x89, 0x3d, 0x38, 0xc9, 0x11, 0x9b, + 0xe4, 0xf7, 0x67, 0xbf, 0xab, 0x6e, 0x0c, 0xa5 }, + .public = { 0xdc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xea, 0xd9, 0xb3, 0x8e, 0xfd, 0xd7, 0x23, 0x63, + 0x79, 0x34, 0xe5, 0x5a, 0xb7, 0x17, 0xa7, 0xae, + 0x09, 0xeb, 0x86, 0xa2, 0x1d, 0xc3, 0x6a, 0x3f, + 0xee, 0xb8, 0x8b, 0x75, 0x9e, 0x39, 0x1e, 0x09 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x90, 0x1b, 0x9d, 0xcf, 0x88, 0x1e, 0x01, 0xe0, + 0x27, 0x57, 0x50, 0x35, 0xd4, 0x0b, 0x43, 0xbd, + 0xc1, 0xc5, 0x24, 0x2e, 0x03, 0x08, 0x47, 0x49, + 0x5b, 0x0c, 0x72, 0x86, 0x46, 0x9b, 0x65, 0x91 }, + .public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x60, 0x2f, 0xf4, 0x07, 0x89, 0xb5, 0x4b, 0x41, + 0x80, 0x59, 0x15, 0xfe, 0x2a, 0x62, 0x21, 0xf0, + 0x7a, 0x50, 0xff, 0xc2, 0xc3, 0xfc, 0x94, 0xcf, + 0x61, 0xf1, 0x3d, 0x79, 0x04, 0xe8, 0x8e, 0x0e }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x80, 0x46, 0x67, 0x7c, 0x28, 0xfd, 0x82, 0xc9, + 0xa1, 0xbd, 0xb7, 0x1a, 0x1a, 0x1a, 0x34, 0xfa, + 0xba, 0x12, 0x25, 0xe2, 0x50, 0x7f, 0xe3, 0xf5, + 0x4d, 0x10, 0xbd, 0x5b, 0x0d, 0x86, 0x5f, 0x8e }, + .public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xe0, 0x0a, 0xe8, 0xb1, 0x43, 0x47, 0x12, 0x47, + 0xba, 0x24, 0xf1, 0x2c, 0x88, 0x55, 0x36, 0xc3, + 0xcb, 0x98, 0x1b, 0x58, 0xe1, 0xe5, 0x6b, 0x2b, + 0xaf, 0x35, 0xc1, 0x2a, 0xe1, 0xf7, 0x9c, 0x26 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x60, 0x2f, 0x7e, 0x2f, 0x68, 0xa8, 0x46, 0xb8, + 0x2c, 0xc2, 0x69, 0xb1, 0xd4, 0x8e, 0x93, 0x98, + 0x86, 0xae, 0x54, 0xfd, 0x63, 0x6c, 0x1f, 0xe0, + 0x74, 0xd7, 0x10, 0x12, 0x7d, 0x47, 0x24, 0x91 }, + .public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x98, 0xcb, 0x9b, 0x50, 0xdd, 0x3f, 0xc2, 0xb0, + 0xd4, 0xf2, 0xd2, 0xbf, 0x7c, 0x5c, 0xfd, 0xd1, + 0x0c, 0x8f, 0xcd, 0x31, 0xfc, 0x40, 0xaf, 0x1a, + 0xd4, 0x4f, 0x47, 0xc1, 0x31, 0x37, 0x63, 0x62 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x60, 0x88, 0x7b, 0x3d, 0xc7, 0x24, 0x43, 0x02, + 0x6e, 0xbe, 0xdb, 0xbb, 0xb7, 0x06, 0x65, 0xf4, + 0x2b, 0x87, 0xad, 0xd1, 0x44, 0x0e, 0x77, 0x68, + 0xfb, 0xd7, 0xe8, 0xe2, 0xce, 0x5f, 0x63, 0x9d }, + .public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x38, 0xd6, 0x30, 0x4c, 0x4a, 0x7e, 0x6d, 0x9f, + 0x79, 0x59, 0x33, 0x4f, 0xb5, 0x24, 0x5b, 0xd2, + 0xc7, 0x54, 0x52, 0x5d, 0x4c, 0x91, 0xdb, 0x95, + 0x02, 0x06, 0x92, 0x62, 0x34, 0xc1, 0xf6, 0x33 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x78, 0xd3, 0x1d, 0xfa, 0x85, 0x44, 0x97, 0xd7, + 0x2d, 0x8d, 0xef, 0x8a, 0x1b, 0x7f, 0xb0, 0x06, + 0xce, 0xc2, 0xd8, 0xc4, 0x92, 0x46, 0x47, 0xc9, + 0x38, 0x14, 0xae, 0x56, 0xfa, 0xed, 0xa4, 0x95 }, + .public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x78, 0x6c, 0xd5, 0x49, 0x96, 0xf0, 0x14, 0xa5, + 0xa0, 0x31, 0xec, 0x14, 0xdb, 0x81, 0x2e, 0xd0, + 0x83, 0x55, 0x06, 0x1f, 0xdb, 0x5d, 0xe6, 0x80, + 0xa8, 0x00, 0xac, 0x52, 0x1f, 0x31, 0x8e, 0x23 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xc0, 0x4c, 0x5b, 0xae, 0xfa, 0x83, 0x02, 0xdd, + 0xde, 0xd6, 0xa4, 0xbb, 0x95, 0x77, 0x61, 0xb4, + 0xeb, 0x97, 0xae, 0xfa, 0x4f, 0xc3, 0xb8, 0x04, + 0x30, 0x85, 0xf9, 0x6a, 0x56, 0x59, 0xb3, 0xa5 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x29, 0xae, 0x8b, 0xc7, 0x3e, 0x9b, 0x10, 0xa0, + 0x8b, 0x4f, 0x68, 0x1c, 0x43, 0xc3, 0xe0, 0xac, + 0x1a, 0x17, 0x1d, 0x31, 0xb3, 0x8f, 0x1a, 0x48, + 0xef, 0xba, 0x29, 0xae, 0x63, 0x9e, 0xa1, 0x34 }, + .valid = true + }, + /* wycheproof - RFC 7748 */ + { + .private = { 0xa0, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, + 0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, + 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18, + 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0x44 }, + .public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, + 0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, + 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b, + 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c }, + .result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, + 0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, + 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7, + 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 }, + .valid = true + }, + /* wycheproof - RFC 7748 */ + { + .private = { 0x48, 0x66, 0xe9, 0xd4, 0xd1, 0xb4, 0x67, 0x3c, + 0x5a, 0xd2, 0x26, 0x91, 0x95, 0x7d, 0x6a, 0xf5, + 0xc1, 0x1b, 0x64, 0x21, 0xe0, 0xea, 0x01, 0xd4, + 0x2c, 0xa4, 0x16, 0x9e, 0x79, 0x18, 0xba, 0x4d }, + .public = { 0xe5, 0x21, 0x0f, 0x12, 0x78, 0x68, 0x11, 0xd3, + 0xf4, 0xb7, 0x95, 0x9d, 0x05, 0x38, 0xae, 0x2c, + 0x31, 0xdb, 0xe7, 0x10, 0x6f, 0xc0, 0x3c, 0x3e, + 0xfc, 0x4c, 0xd5, 0x49, 0xc7, 0x15, 0xa4, 0x13 }, + .result = { 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d, + 0x7a, 0xad, 0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8, + 0x8b, 0x59, 0x5a, 0x68, 0x79, 0x9f, 0xa1, 0x52, + 0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac, 0x79, 0x57 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x0a, 0xb4, 0xe7, 0x63, 0x80, 0xd8, 0x4d, 0xde, + 0x4f, 0x68, 0x33, 0xc5, 0x8f, 0x2a, 0x9f, 0xb8, + 0xf8, 0x3b, 0xb0, 0x16, 0x9b, 0x17, 0x2b, 0xe4, + 0xb6, 0xe0, 0x59, 0x28, 0x87, 0x74, 0x1a, 0x36 }, + .result = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x89, 0xe1, 0x0d, 0x57, 0x01, 0xb4, 0x33, 0x7d, + 0x2d, 0x03, 0x21, 0x81, 0x53, 0x8b, 0x10, 0x64, + 0xbd, 0x40, 0x84, 0x40, 0x1c, 0xec, 0xa1, 0xfd, + 0x12, 0x66, 0x3a, 0x19, 0x59, 0x38, 0x80, 0x00 }, + .result = { 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x2b, 0x55, 0xd3, 0xaa, 0x4a, 0x8f, 0x80, 0xc8, + 0xc0, 0xb2, 0xae, 0x5f, 0x93, 0x3e, 0x85, 0xaf, + 0x49, 0xbe, 0xac, 0x36, 0xc2, 0xfa, 0x73, 0x94, + 0xba, 0xb7, 0x6c, 0x89, 0x33, 0xf8, 0xf8, 0x1d }, + .result = { 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x63, 0xe5, 0xb1, 0xfe, 0x96, 0x01, 0xfe, 0x84, + 0x38, 0x5d, 0x88, 0x66, 0xb0, 0x42, 0x12, 0x62, + 0xf7, 0x8f, 0xbf, 0xa5, 0xaf, 0xf9, 0x58, 0x5e, + 0x62, 0x66, 0x79, 0xb1, 0x85, 0x47, 0xd9, 0x59 }, + .result = { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xe4, 0x28, 0xf3, 0xda, 0xc1, 0x78, 0x09, 0xf8, + 0x27, 0xa5, 0x22, 0xce, 0x32, 0x35, 0x50, 0x58, + 0xd0, 0x73, 0x69, 0x36, 0x4a, 0xa7, 0x89, 0x02, + 0xee, 0x10, 0x13, 0x9b, 0x9f, 0x9d, 0xd6, 0x53 }, + .result = { 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xb3, 0xb5, 0x0e, 0x3e, 0xd3, 0xa4, 0x07, 0xb9, + 0x5d, 0xe9, 0x42, 0xef, 0x74, 0x57, 0x5b, 0x5a, + 0xb8, 0xa1, 0x0c, 0x09, 0xee, 0x10, 0x35, 0x44, + 0xd6, 0x0b, 0xdf, 0xed, 0x81, 0x38, 0xab, 0x2b }, + .result = { 0xf9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x21, 0x3f, 0xff, 0xe9, 0x3d, 0x5e, 0xa8, 0xcd, + 0x24, 0x2e, 0x46, 0x28, 0x44, 0x02, 0x99, 0x22, + 0xc4, 0x3c, 0x77, 0xc9, 0xe3, 0xe4, 0x2f, 0x56, + 0x2f, 0x48, 0x5d, 0x24, 0xc5, 0x01, 0xa2, 0x0b }, + .result = { 0xf3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x91, 0xb2, 0x32, 0xa1, 0x78, 0xb3, 0xcd, 0x53, + 0x09, 0x32, 0x44, 0x1e, 0x61, 0x39, 0x41, 0x8f, + 0x72, 0x17, 0x22, 0x92, 0xf1, 0xda, 0x4c, 0x18, + 0x34, 0xfc, 0x5e, 0xbf, 0xef, 0xb5, 0x1e, 0x3f }, + .result = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x04, 0x5c, 0x6e, 0x11, 0xc5, 0xd3, 0x32, 0x55, + 0x6c, 0x78, 0x22, 0xfe, 0x94, 0xeb, 0xf8, 0x9b, + 0x56, 0xa3, 0x87, 0x8d, 0xc2, 0x7c, 0xa0, 0x79, + 0x10, 0x30, 0x58, 0x84, 0x9f, 0xab, 0xcb, 0x4f }, + .result = { 0xe5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x1c, 0xa2, 0x19, 0x0b, 0x71, 0x16, 0x35, 0x39, + 0x06, 0x3c, 0x35, 0x77, 0x3b, 0xda, 0x0c, 0x9c, + 0x92, 0x8e, 0x91, 0x36, 0xf0, 0x62, 0x0a, 0xeb, + 0x09, 0x3f, 0x09, 0x91, 0x97, 0xb7, 0xf7, 0x4e }, + .result = { 0xe3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xf7, 0x6e, 0x90, 0x10, 0xac, 0x33, 0xc5, 0x04, + 0x3b, 0x2d, 0x3b, 0x76, 0xa8, 0x42, 0x17, 0x10, + 0x00, 0xc4, 0x91, 0x62, 0x22, 0xe9, 0xe8, 0x58, + 0x97, 0xa0, 0xae, 0xc7, 0xf6, 0x35, 0x0b, 0x3c }, + .result = { 0xdd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xbb, 0x72, 0x68, 0x8d, 0x8f, 0x8a, 0xa7, 0xa3, + 0x9c, 0xd6, 0x06, 0x0c, 0xd5, 0xc8, 0x09, 0x3c, + 0xde, 0xc6, 0xfe, 0x34, 0x19, 0x37, 0xc3, 0x88, + 0x6a, 0x99, 0x34, 0x6c, 0xd0, 0x7f, 0xaa, 0x55 }, + .result = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x88, 0xfd, 0xde, 0xa1, 0x93, 0x39, 0x1c, 0x6a, + 0x59, 0x33, 0xef, 0x9b, 0x71, 0x90, 0x15, 0x49, + 0x44, 0x72, 0x05, 0xaa, 0xe9, 0xda, 0x92, 0x8a, + 0x6b, 0x91, 0xa3, 0x52, 0xba, 0x10, 0xf4, 0x1f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x30, 0x3b, 0x39, 0x2f, 0x15, 0x31, 0x16, 0xca, + 0xd9, 0xcc, 0x68, 0x2a, 0x00, 0xcc, 0xc4, 0x4c, + 0x95, 0xff, 0x0d, 0x3b, 0xbe, 0x56, 0x8b, 0xeb, + 0x6c, 0x4e, 0x73, 0x9b, 0xaf, 0xdc, 0x2c, 0x68 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xfd, 0x30, 0x0a, 0xeb, 0x40, 0xe1, 0xfa, 0x58, + 0x25, 0x18, 0x41, 0x2b, 0x49, 0xb2, 0x08, 0xa7, + 0x84, 0x2b, 0x1e, 0x1f, 0x05, 0x6a, 0x04, 0x01, + 0x78, 0xea, 0x41, 0x41, 0x53, 0x4f, 0x65, 0x2d }, + .result = { 0xb7, 0x34, 0x10, 0x5d, 0xc2, 0x57, 0x58, 0x5d, + 0x73, 0xb5, 0x66, 0xcc, 0xb7, 0x6f, 0x06, 0x27, + 0x95, 0xcc, 0xbe, 0xc8, 0x91, 0x28, 0xe5, 0x2b, + 0x02, 0xf3, 0xe5, 0x96, 0x39, 0xf1, 0x3c, 0x46 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xc8, 0xef, 0x79, 0xb5, 0x14, 0xd7, 0x68, 0x26, + 0x77, 0xbc, 0x79, 0x31, 0xe0, 0x6e, 0xe5, 0xc2, + 0x7c, 0x9b, 0x39, 0x2b, 0x4a, 0xe9, 0x48, 0x44, + 0x73, 0xf5, 0x54, 0xe6, 0x67, 0x8e, 0xcc, 0x2e }, + .result = { 0x64, 0x7a, 0x46, 0xb6, 0xfc, 0x3f, 0x40, 0xd6, + 0x21, 0x41, 0xee, 0x3c, 0xee, 0x70, 0x6b, 0x4d, + 0x7a, 0x92, 0x71, 0x59, 0x3a, 0x7b, 0x14, 0x3e, + 0x8e, 0x2e, 0x22, 0x79, 0x88, 0x3e, 0x45, 0x50 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0x64, 0xae, 0xac, 0x25, 0x04, 0x14, 0x48, 0x61, + 0x53, 0x2b, 0x7b, 0xbc, 0xb6, 0xc8, 0x7d, 0x67, + 0xdd, 0x4c, 0x1f, 0x07, 0xeb, 0xc2, 0xe0, 0x6e, + 0xff, 0xb9, 0x5a, 0xec, 0xc6, 0x17, 0x0b, 0x2c }, + .result = { 0x4f, 0xf0, 0x3d, 0x5f, 0xb4, 0x3c, 0xd8, 0x65, + 0x7a, 0x3c, 0xf3, 0x7c, 0x13, 0x8c, 0xad, 0xce, + 0xcc, 0xe5, 0x09, 0xe4, 0xeb, 0xa0, 0x89, 0xd0, + 0xef, 0x40, 0xb4, 0xe4, 0xfb, 0x94, 0x61, 0x55 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xbf, 0x68, 0xe3, 0x5e, 0x9b, 0xdb, 0x7e, 0xee, + 0x1b, 0x50, 0x57, 0x02, 0x21, 0x86, 0x0f, 0x5d, + 0xcd, 0xad, 0x8a, 0xcb, 0xab, 0x03, 0x1b, 0x14, + 0x97, 0x4c, 0xc4, 0x90, 0x13, 0xc4, 0x98, 0x31 }, + .result = { 0x21, 0xce, 0xe5, 0x2e, 0xfd, 0xbc, 0x81, 0x2e, + 0x1d, 0x02, 0x1a, 0x4a, 0xf1, 0xe1, 0xd8, 0xbc, + 0x4d, 0xb3, 0xc4, 0x00, 0xe4, 0xd2, 0xa2, 0xc5, + 0x6a, 0x39, 0x26, 0xdb, 0x4d, 0x99, 0xc6, 0x5b }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0x53, 0x47, 0xc4, 0x91, 0x33, 0x1a, 0x64, 0xb4, + 0x3d, 0xdc, 0x68, 0x30, 0x34, 0xe6, 0x77, 0xf5, + 0x3d, 0xc3, 0x2b, 0x52, 0xa5, 0x2a, 0x57, 0x7c, + 0x15, 0xa8, 0x3b, 0xf2, 0x98, 0xe9, 0x9f, 0x19 }, + .result = { 0x18, 0xcb, 0x89, 0xe4, 0xe2, 0x0c, 0x0c, 0x2b, + 0xd3, 0x24, 0x30, 0x52, 0x45, 0x26, 0x6c, 0x93, + 0x27, 0x69, 0x0b, 0xbe, 0x79, 0xac, 0xb8, 0x8f, + 0x5b, 0x8f, 0xb3, 0xf7, 0x4e, 0xca, 0x3e, 0x52 }, + .valid = true + }, + /* wycheproof - private key == -1 (mod order) */ + { + .private = { 0xa0, 0x23, 0xcd, 0xd0, 0x83, 0xef, 0x5b, 0xb8, + 0x2f, 0x10, 0xd6, 0x2e, 0x59, 0xe1, 0x5a, 0x68, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50 }, + .public = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e, + 0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57, + 0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f, + 0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 }, + .result = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e, + 0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57, + 0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f, + 0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 }, + .valid = true + }, + /* wycheproof - private key == 1 (mod order) on twist */ + { + .private = { 0x58, 0x08, 0x3d, 0xd2, 0x61, 0xad, 0x91, 0xef, + 0xf9, 0x52, 0x32, 0x2e, 0xc8, 0x24, 0xc6, 0x82, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f }, + .public = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f, + 0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6, + 0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64, + 0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 }, + .result = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f, + 0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6, + 0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64, + 0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 }, + .valid = true + } +}; + +bool __init curve25519_selftest(void) +{ + bool success = true, ret, ret2; + size_t i = 0, j; + u8 in[CURVE25519_KEY_SIZE]; + u8 out[CURVE25519_KEY_SIZE], out2[CURVE25519_KEY_SIZE], + out3[CURVE25519_KEY_SIZE]; + + for (i = 0; i < ARRAY_SIZE(curve25519_test_vectors); ++i) { + memset(out, 0, CURVE25519_KEY_SIZE); + ret = curve25519(out, curve25519_test_vectors[i].private, + curve25519_test_vectors[i].public); + if (ret != curve25519_test_vectors[i].valid || + memcmp(out, curve25519_test_vectors[i].result, + CURVE25519_KEY_SIZE)) { + pr_err("curve25519 self-test %zu: FAIL\n", i + 1); + success = false; + } + } + + for (i = 0; i < 5; ++i) { + get_random_bytes(in, sizeof(in)); + ret = curve25519_generate_public(out, in); + ret2 = curve25519(out2, in, (u8[CURVE25519_KEY_SIZE]){ 9 }); + curve25519_generic(out3, in, (u8[CURVE25519_KEY_SIZE]){ 9 }); + if (ret != ret2 || + memcmp(out, out2, CURVE25519_KEY_SIZE) || + memcmp(out, out3, CURVE25519_KEY_SIZE)) { + pr_err("curve25519 basepoint self-test %zu: FAIL: input - 0x", + i + 1); + for (j = CURVE25519_KEY_SIZE; j-- > 0;) + printk(KERN_CONT "%02x", in[j]); + printk(KERN_CONT "\n"); + success = false; + } + } + + return success; +} diff --git a/lib/crypto/curve25519.c b/lib/crypto/curve25519.c index 0106bebe6900..064b352c6907 100644 --- a/lib/crypto/curve25519.c +++ b/lib/crypto/curve25519.c @@ -13,12 +13,20 @@ #include <linux/module.h> #include <linux/init.h> -const u8 curve25519_null_point[CURVE25519_KEY_SIZE] __aligned(32) = { 0 }; -const u8 curve25519_base_point[CURVE25519_KEY_SIZE] __aligned(32) = { 9 }; +static int __init curve25519_init(void) +{ + if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && + WARN_ON(!curve25519_selftest())) + return -ENODEV; + return 0; +} -EXPORT_SYMBOL(curve25519_null_point); -EXPORT_SYMBOL(curve25519_base_point); -EXPORT_SYMBOL(curve25519_generic); +static void __exit curve25519_exit(void) +{ +} + +module_init(curve25519_init); +module_exit(curve25519_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Curve25519 scalar multiplication"); diff --git a/lib/crypto/poly1305-donna32.c b/lib/crypto/poly1305-donna32.c new file mode 100644 index 000000000000..7fb71845cc84 --- /dev/null +++ b/lib/crypto/poly1305-donna32.c @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +#include <linux/kernel.h> +#include <asm/unaligned.h> +#include <crypto/internal/poly1305.h> + +void poly1305_core_setkey(struct poly1305_core_key *key, + const u8 raw_key[POLY1305_BLOCK_SIZE]) +{ + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + key->key.r[0] = (get_unaligned_le32(&raw_key[0])) & 0x3ffffff; + key->key.r[1] = (get_unaligned_le32(&raw_key[3]) >> 2) & 0x3ffff03; + key->key.r[2] = (get_unaligned_le32(&raw_key[6]) >> 4) & 0x3ffc0ff; + key->key.r[3] = (get_unaligned_le32(&raw_key[9]) >> 6) & 0x3f03fff; + key->key.r[4] = (get_unaligned_le32(&raw_key[12]) >> 8) & 0x00fffff; + + /* s = 5*r */ + key->precomputed_s.r[0] = key->key.r[1] * 5; + key->precomputed_s.r[1] = key->key.r[2] * 5; + key->precomputed_s.r[2] = key->key.r[3] * 5; + key->precomputed_s.r[3] = key->key.r[4] * 5; +} +EXPORT_SYMBOL(poly1305_core_setkey); + +void poly1305_core_blocks(struct poly1305_state *state, + const struct poly1305_core_key *key, const void *src, + unsigned int nblocks, u32 hibit) +{ + const u8 *input = src; + u32 r0, r1, r2, r3, r4; + u32 s1, s2, s3, s4; + u32 h0, h1, h2, h3, h4; + u64 d0, d1, d2, d3, d4; + u32 c; + + if (!nblocks) + return; + + hibit <<= 24; + + r0 = key->key.r[0]; + r1 = key->key.r[1]; + r2 = key->key.r[2]; + r3 = key->key.r[3]; + r4 = key->key.r[4]; + + s1 = key->precomputed_s.r[0]; + s2 = key->precomputed_s.r[1]; + s3 = key->precomputed_s.r[2]; + s4 = key->precomputed_s.r[3]; + + h0 = state->h[0]; + h1 = state->h[1]; + h2 = state->h[2]; + h3 = state->h[3]; + h4 = state->h[4]; + + do { + /* h += m[i] */ + h0 += (get_unaligned_le32(&input[0])) & 0x3ffffff; + h1 += (get_unaligned_le32(&input[3]) >> 2) & 0x3ffffff; + h2 += (get_unaligned_le32(&input[6]) >> 4) & 0x3ffffff; + h3 += (get_unaligned_le32(&input[9]) >> 6) & 0x3ffffff; + h4 += (get_unaligned_le32(&input[12]) >> 8) | hibit; + + /* h *= r */ + d0 = ((u64)h0 * r0) + ((u64)h1 * s4) + + ((u64)h2 * s3) + ((u64)h3 * s2) + + ((u64)h4 * s1); + d1 = ((u64)h0 * r1) + ((u64)h1 * r0) + + ((u64)h2 * s4) + ((u64)h3 * s3) + + ((u64)h4 * s2); + d2 = ((u64)h0 * r2) + ((u64)h1 * r1) + + ((u64)h2 * r0) + ((u64)h3 * s4) + + ((u64)h4 * s3); + d3 = ((u64)h0 * r3) + ((u64)h1 * r2) + + ((u64)h2 * r1) + ((u64)h3 * r0) + + ((u64)h4 * s4); + d4 = ((u64)h0 * r4) + ((u64)h1 * r3) + + ((u64)h2 * r2) + ((u64)h3 * r1) + + ((u64)h4 * r0); + + /* (partial) h %= p */ + c = (u32)(d0 >> 26); + h0 = (u32)d0 & 0x3ffffff; + d1 += c; + c = (u32)(d1 >> 26); + h1 = (u32)d1 & 0x3ffffff; + d2 += c; + c = (u32)(d2 >> 26); + h2 = (u32)d2 & 0x3ffffff; + d3 += c; + c = (u32)(d3 >> 26); + h3 = (u32)d3 & 0x3ffffff; + d4 += c; + c = (u32)(d4 >> 26); + h4 = (u32)d4 & 0x3ffffff; + h0 += c * 5; + c = (h0 >> 26); + h0 = h0 & 0x3ffffff; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + } while (--nblocks); + + state->h[0] = h0; + state->h[1] = h1; + state->h[2] = h2; + state->h[3] = h3; + state->h[4] = h4; +} +EXPORT_SYMBOL(poly1305_core_blocks); + +void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], + void *dst) +{ + u8 *mac = dst; + u32 h0, h1, h2, h3, h4, c; + u32 g0, g1, g2, g3, g4; + u64 f; + u32 mask; + + /* fully carry h */ + h0 = state->h[0]; + h1 = state->h[1]; + h2 = state->h[2]; + h3 = state->h[3]; + h4 = state->h[4]; + + c = h1 >> 26; + h1 = h1 & 0x3ffffff; + h2 += c; + c = h2 >> 26; + h2 = h2 & 0x3ffffff; + h3 += c; + c = h3 >> 26; + h3 = h3 & 0x3ffffff; + h4 += c; + c = h4 >> 26; + h4 = h4 & 0x3ffffff; + h0 += c * 5; + c = h0 >> 26; + h0 = h0 & 0x3ffffff; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 26; + g0 &= 0x3ffffff; + g1 = h1 + c; + c = g1 >> 26; + g1 &= 0x3ffffff; + g2 = h2 + c; + c = g2 >> 26; + g2 &= 0x3ffffff; + g3 = h3 + c; + c = g3 >> 26; + g3 &= 0x3ffffff; + g4 = h4 + c - (1UL << 26); + + /* select h if h < p, or h + -p if h >= p */ + mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; + g0 &= mask; + g1 &= mask; + g2 &= mask; + g3 &= mask; + g4 &= mask; + mask = ~mask; + + h0 = (h0 & mask) | g0; + h1 = (h1 & mask) | g1; + h2 = (h2 & mask) | g2; + h3 = (h3 & mask) | g3; + h4 = (h4 & mask) | g4; + + /* h = h % (2^128) */ + h0 = ((h0) | (h1 << 26)) & 0xffffffff; + h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff; + h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff; + h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff; + + if (likely(nonce)) { + /* mac = (h + nonce) % (2^128) */ + f = (u64)h0 + nonce[0]; + h0 = (u32)f; + f = (u64)h1 + nonce[1] + (f >> 32); + h1 = (u32)f; + f = (u64)h2 + nonce[2] + (f >> 32); + h2 = (u32)f; + f = (u64)h3 + nonce[3] + (f >> 32); + h3 = (u32)f; + } + + put_unaligned_le32(h0, &mac[0]); + put_unaligned_le32(h1, &mac[4]); + put_unaligned_le32(h2, &mac[8]); + put_unaligned_le32(h3, &mac[12]); +} +EXPORT_SYMBOL(poly1305_core_emit); diff --git a/lib/crypto/poly1305-donna64.c b/lib/crypto/poly1305-donna64.c new file mode 100644 index 000000000000..d34cf4053668 --- /dev/null +++ b/lib/crypto/poly1305-donna64.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +#include <linux/kernel.h> +#include <asm/unaligned.h> +#include <crypto/internal/poly1305.h> + +typedef __uint128_t u128; + +void poly1305_core_setkey(struct poly1305_core_key *key, + const u8 raw_key[POLY1305_BLOCK_SIZE]) +{ + u64 t0, t1; + + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + t0 = get_unaligned_le64(&raw_key[0]); + t1 = get_unaligned_le64(&raw_key[8]); + + key->key.r64[0] = t0 & 0xffc0fffffffULL; + key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL; + key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL; + + /* s = 20*r */ + key->precomputed_s.r64[0] = key->key.r64[1] * 20; + key->precomputed_s.r64[1] = key->key.r64[2] * 20; +} +EXPORT_SYMBOL(poly1305_core_setkey); + +void poly1305_core_blocks(struct poly1305_state *state, + const struct poly1305_core_key *key, const void *src, + unsigned int nblocks, u32 hibit) +{ + const u8 *input = src; + u64 hibit64; + u64 r0, r1, r2; + u64 s1, s2; + u64 h0, h1, h2; + u64 c; + u128 d0, d1, d2, d; + + if (!nblocks) + return; + + hibit64 = ((u64)hibit) << 40; + + r0 = key->key.r64[0]; + r1 = key->key.r64[1]; + r2 = key->key.r64[2]; + + h0 = state->h64[0]; + h1 = state->h64[1]; + h2 = state->h64[2]; + + s1 = key->precomputed_s.r64[0]; + s2 = key->precomputed_s.r64[1]; + + do { + u64 t0, t1; + + /* h += m[i] */ + t0 = get_unaligned_le64(&input[0]); + t1 = get_unaligned_le64(&input[8]); + + h0 += t0 & 0xfffffffffffULL; + h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64; + + /* h *= r */ + d0 = (u128)h0 * r0; + d = (u128)h1 * s2; + d0 += d; + d = (u128)h2 * s1; + d0 += d; + d1 = (u128)h0 * r1; + d = (u128)h1 * r0; + d1 += d; + d = (u128)h2 * s2; + d1 += d; + d2 = (u128)h0 * r2; + d = (u128)h1 * r1; + d2 += d; + d = (u128)h2 * r0; + d2 += d; + + /* (partial) h %= p */ + c = (u64)(d0 >> 44); + h0 = (u64)d0 & 0xfffffffffffULL; + d1 += c; + c = (u64)(d1 >> 44); + h1 = (u64)d1 & 0xfffffffffffULL; + d2 += c; + c = (u64)(d2 >> 42); + h2 = (u64)d2 & 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 = h0 & 0xfffffffffffULL; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + } while (--nblocks); + + state->h64[0] = h0; + state->h64[1] = h1; + state->h64[2] = h2; +} +EXPORT_SYMBOL(poly1305_core_blocks); + +void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], + void *dst) +{ + u8 *mac = dst; + u64 h0, h1, h2, c; + u64 g0, g1, g2; + u64 t0, t1; + + /* fully carry h */ + h0 = state->h64[0]; + h1 = state->h64[1]; + h2 = state->h64[2]; + + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 44; + g0 &= 0xfffffffffffULL; + g1 = h1 + c; + c = g1 >> 44; + g1 &= 0xfffffffffffULL; + g2 = h2 + c - (1ULL << 42); + + /* select h if h < p, or h + -p if h >= p */ + c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1; + g0 &= c; + g1 &= c; + g2 &= c; + c = ~c; + h0 = (h0 & c) | g0; + h1 = (h1 & c) | g1; + h2 = (h2 & c) | g2; + + if (likely(nonce)) { + /* h = (h + nonce) */ + t0 = ((u64)nonce[1] << 32) | nonce[0]; + t1 = ((u64)nonce[3] << 32) | nonce[2]; + + h0 += t0 & 0xfffffffffffULL; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c; + h2 &= 0x3ffffffffffULL; + } + + /* mac = h % (2^128) */ + h0 = h0 | (h1 << 44); + h1 = (h1 >> 20) | (h2 << 24); + + put_unaligned_le64(h0, &mac[0]); + put_unaligned_le64(h1, &mac[8]); +} +EXPORT_SYMBOL(poly1305_core_emit); diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 32ec293c65ae..26d87fc3823e 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -12,151 +12,10 @@ #include <linux/module.h> #include <asm/unaligned.h> -static inline u64 mlt(u64 a, u64 b) +void poly1305_init_generic(struct poly1305_desc_ctx *desc, + const u8 key[POLY1305_KEY_SIZE]) { - return a * b; -} - -static inline u32 sr(u64 v, u_char n) -{ - return v >> n; -} - -static inline u32 and(u32 v, u32 mask) -{ - return v & mask; -} - -void poly1305_core_setkey(struct poly1305_key *key, const u8 *raw_key) -{ - /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ - key->r[0] = (get_unaligned_le32(raw_key + 0) >> 0) & 0x3ffffff; - key->r[1] = (get_unaligned_le32(raw_key + 3) >> 2) & 0x3ffff03; - key->r[2] = (get_unaligned_le32(raw_key + 6) >> 4) & 0x3ffc0ff; - key->r[3] = (get_unaligned_le32(raw_key + 9) >> 6) & 0x3f03fff; - key->r[4] = (get_unaligned_le32(raw_key + 12) >> 8) & 0x00fffff; -} -EXPORT_SYMBOL_GPL(poly1305_core_setkey); - -void poly1305_core_blocks(struct poly1305_state *state, - const struct poly1305_key *key, const void *src, - unsigned int nblocks, u32 hibit) -{ - u32 r0, r1, r2, r3, r4; - u32 s1, s2, s3, s4; - u32 h0, h1, h2, h3, h4; - u64 d0, d1, d2, d3, d4; - - if (!nblocks) - return; - - r0 = key->r[0]; - r1 = key->r[1]; - r2 = key->r[2]; - r3 = key->r[3]; - r4 = key->r[4]; - - s1 = r1 * 5; - s2 = r2 * 5; - s3 = r3 * 5; - s4 = r4 * 5; - - h0 = state->h[0]; - h1 = state->h[1]; - h2 = state->h[2]; - h3 = state->h[3]; - h4 = state->h[4]; - - do { - /* h += m[i] */ - h0 += (get_unaligned_le32(src + 0) >> 0) & 0x3ffffff; - h1 += (get_unaligned_le32(src + 3) >> 2) & 0x3ffffff; - h2 += (get_unaligned_le32(src + 6) >> 4) & 0x3ffffff; - h3 += (get_unaligned_le32(src + 9) >> 6) & 0x3ffffff; - h4 += (get_unaligned_le32(src + 12) >> 8) | (hibit << 24); - - /* h *= r */ - d0 = mlt(h0, r0) + mlt(h1, s4) + mlt(h2, s3) + - mlt(h3, s2) + mlt(h4, s1); - d1 = mlt(h0, r1) + mlt(h1, r0) + mlt(h2, s4) + - mlt(h3, s3) + mlt(h4, s2); - d2 = mlt(h0, r2) + mlt(h1, r1) + mlt(h2, r0) + - mlt(h3, s4) + mlt(h4, s3); - d3 = mlt(h0, r3) + mlt(h1, r2) + mlt(h2, r1) + - mlt(h3, r0) + mlt(h4, s4); - d4 = mlt(h0, r4) + mlt(h1, r3) + mlt(h2, r2) + - mlt(h3, r1) + mlt(h4, r0); - - /* (partial) h %= p */ - d1 += sr(d0, 26); h0 = and(d0, 0x3ffffff); - d2 += sr(d1, 26); h1 = and(d1, 0x3ffffff); - d3 += sr(d2, 26); h2 = and(d2, 0x3ffffff); - d4 += sr(d3, 26); h3 = and(d3, 0x3ffffff); - h0 += sr(d4, 26) * 5; h4 = and(d4, 0x3ffffff); - h1 += h0 >> 26; h0 = h0 & 0x3ffffff; - - src += POLY1305_BLOCK_SIZE; - } while (--nblocks); - - state->h[0] = h0; - state->h[1] = h1; - state->h[2] = h2; - state->h[3] = h3; - state->h[4] = h4; -} -EXPORT_SYMBOL_GPL(poly1305_core_blocks); - -void poly1305_core_emit(const struct poly1305_state *state, void *dst) -{ - u32 h0, h1, h2, h3, h4; - u32 g0, g1, g2, g3, g4; - u32 mask; - - /* fully carry h */ - h0 = state->h[0]; - h1 = state->h[1]; - h2 = state->h[2]; - h3 = state->h[3]; - h4 = state->h[4]; - - h2 += (h1 >> 26); h1 = h1 & 0x3ffffff; - h3 += (h2 >> 26); h2 = h2 & 0x3ffffff; - h4 += (h3 >> 26); h3 = h3 & 0x3ffffff; - h0 += (h4 >> 26) * 5; h4 = h4 & 0x3ffffff; - h1 += (h0 >> 26); h0 = h0 & 0x3ffffff; - - /* compute h + -p */ - g0 = h0 + 5; - g1 = h1 + (g0 >> 26); g0 &= 0x3ffffff; - g2 = h2 + (g1 >> 26); g1 &= 0x3ffffff; - g3 = h3 + (g2 >> 26); g2 &= 0x3ffffff; - g4 = h4 + (g3 >> 26) - (1 << 26); g3 &= 0x3ffffff; - - /* select h if h < p, or h + -p if h >= p */ - mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; - g0 &= mask; - g1 &= mask; - g2 &= mask; - g3 &= mask; - g4 &= mask; - mask = ~mask; - h0 = (h0 & mask) | g0; - h1 = (h1 & mask) | g1; - h2 = (h2 & mask) | g2; - h3 = (h3 & mask) | g3; - h4 = (h4 & mask) | g4; - - /* h = h % (2^128) */ - put_unaligned_le32((h0 >> 0) | (h1 << 26), dst + 0); - put_unaligned_le32((h1 >> 6) | (h2 << 20), dst + 4); - put_unaligned_le32((h2 >> 12) | (h3 << 14), dst + 8); - put_unaligned_le32((h3 >> 18) | (h4 << 8), dst + 12); -} -EXPORT_SYMBOL_GPL(poly1305_core_emit); - -void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key) -{ - poly1305_core_setkey(desc->r, key); + poly1305_core_setkey(&desc->core_r, key); desc->s[0] = get_unaligned_le32(key + 16); desc->s[1] = get_unaligned_le32(key + 20); desc->s[2] = get_unaligned_le32(key + 24); @@ -164,7 +23,7 @@ void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key) poly1305_core_init(&desc->h); desc->buflen = 0; desc->sset = true; - desc->rset = 1; + desc->rset = 2; } EXPORT_SYMBOL_GPL(poly1305_init_generic); @@ -181,13 +40,14 @@ void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, desc->buflen += bytes; if (desc->buflen == POLY1305_BLOCK_SIZE) { - poly1305_core_blocks(&desc->h, desc->r, desc->buf, 1, 1); + poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, + 1, 1); desc->buflen = 0; } } if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - poly1305_core_blocks(&desc->h, desc->r, src, + poly1305_core_blocks(&desc->h, &desc->core_r, src, nbytes / POLY1305_BLOCK_SIZE, 1); src += nbytes - (nbytes % POLY1305_BLOCK_SIZE); nbytes %= POLY1305_BLOCK_SIZE; @@ -202,28 +62,14 @@ EXPORT_SYMBOL_GPL(poly1305_update_generic); void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst) { - __le32 digest[4]; - u64 f = 0; - if (unlikely(desc->buflen)) { desc->buf[desc->buflen++] = 1; memset(desc->buf + desc->buflen, 0, POLY1305_BLOCK_SIZE - desc->buflen); - poly1305_core_blocks(&desc->h, desc->r, desc->buf, 1, 0); + poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, 1, 0); } - poly1305_core_emit(&desc->h, digest); - - /* mac = (h + s) % (2^128) */ - f = (f >> 32) + le32_to_cpu(digest[0]) + desc->s[0]; - put_unaligned_le32(f, dst + 0); - f = (f >> 32) + le32_to_cpu(digest[1]) + desc->s[1]; - put_unaligned_le32(f, dst + 4); - f = (f >> 32) + le32_to_cpu(digest[2]) + desc->s[2]; - put_unaligned_le32(f, dst + 8); - f = (f >> 32) + le32_to_cpu(digest[3]) + desc->s[3]; - put_unaligned_le32(f, dst + 12); - + poly1305_core_emit(&desc->h, desc->s, dst); *desc = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL_GPL(poly1305_final_generic); diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 66cb04b0cf4e..72a4b0b1df28 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -15,9 +15,28 @@ #include <linux/export.h> #include <linux/module.h> #include <linux/string.h> -#include <crypto/sha.h> +#include <crypto/sha2.h> #include <asm/unaligned.h> +static const u32 SHA256_K[] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + static inline u32 Ch(u32 x, u32 y, u32 z) { return z ^ (x & (y ^ z)); @@ -43,173 +62,68 @@ static inline void BLEND_OP(int I, u32 *W) W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16]; } -static void sha256_transform(u32 *state, const u8 *input) +#define SHA256_ROUND(i, a, b, c, d, e, f, g, h) do { \ + u32 t1, t2; \ + t1 = h + e1(e) + Ch(e, f, g) + SHA256_K[i] + W[i]; \ + t2 = e0(a) + Maj(a, b, c); \ + d += t1; \ + h = t1 + t2; \ +} while (0) + +static void sha256_transform(u32 *state, const u8 *input, u32 *W) { - u32 a, b, c, d, e, f, g, h, t1, t2; - u32 W[64]; + u32 a, b, c, d, e, f, g, h; int i; /* load the input */ - for (i = 0; i < 16; i++) - LOAD_OP(i, W, input); + for (i = 0; i < 16; i += 8) { + LOAD_OP(i + 0, W, input); + LOAD_OP(i + 1, W, input); + LOAD_OP(i + 2, W, input); + LOAD_OP(i + 3, W, input); + LOAD_OP(i + 4, W, input); + LOAD_OP(i + 5, W, input); + LOAD_OP(i + 6, W, input); + LOAD_OP(i + 7, W, input); + } /* now blend */ - for (i = 16; i < 64; i++) - BLEND_OP(i, W); + for (i = 16; i < 64; i += 8) { + BLEND_OP(i + 0, W); + BLEND_OP(i + 1, W); + BLEND_OP(i + 2, W); + BLEND_OP(i + 3, W); + BLEND_OP(i + 4, W); + BLEND_OP(i + 5, W); + BLEND_OP(i + 6, W); + BLEND_OP(i + 7, W); + } /* load the state into our registers */ a = state[0]; b = state[1]; c = state[2]; d = state[3]; e = state[4]; f = state[5]; g = state[6]; h = state[7]; /* now iterate */ - t1 = h + e1(e) + Ch(e, f, g) + 0x428a2f98 + W[0]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x71374491 + W[1]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0xb5c0fbcf + W[2]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0xe9b5dba5 + W[3]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x3956c25b + W[4]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x59f111f1 + W[5]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x923f82a4 + W[6]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0xab1c5ed5 + W[7]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0xd807aa98 + W[8]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x12835b01 + W[9]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x243185be + W[10]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x550c7dc3 + W[11]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x72be5d74 + W[12]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x80deb1fe + W[13]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x9bdc06a7 + W[14]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0xc19bf174 + W[15]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0xe49b69c1 + W[16]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0xefbe4786 + W[17]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x0fc19dc6 + W[18]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x240ca1cc + W[19]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x2de92c6f + W[20]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x4a7484aa + W[21]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x5cb0a9dc + W[22]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x76f988da + W[23]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x983e5152 + W[24]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0xa831c66d + W[25]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0xb00327c8 + W[26]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0xbf597fc7 + W[27]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0xc6e00bf3 + W[28]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0xd5a79147 + W[29]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x06ca6351 + W[30]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x14292967 + W[31]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x27b70a85 + W[32]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x2e1b2138 + W[33]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x4d2c6dfc + W[34]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x53380d13 + W[35]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x650a7354 + W[36]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x766a0abb + W[37]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x81c2c92e + W[38]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x92722c85 + W[39]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0xa2bfe8a1 + W[40]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0xa81a664b + W[41]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0xc24b8b70 + W[42]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0xc76c51a3 + W[43]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0xd192e819 + W[44]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0xd6990624 + W[45]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0xf40e3585 + W[46]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x106aa070 + W[47]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x19a4c116 + W[48]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x1e376c08 + W[49]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x2748774c + W[50]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x34b0bcb5 + W[51]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x391c0cb3 + W[52]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x4ed8aa4a + W[53]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x5b9cca4f + W[54]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x682e6ff3 + W[55]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x748f82ee + W[56]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x78a5636f + W[57]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x84c87814 + W[58]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x8cc70208 + W[59]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x90befffa + W[60]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0xa4506ceb + W[61]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0xbef9a3f7 + W[62]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0xc67178f2 + W[63]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; + for (i = 0; i < 64; i += 8) { + SHA256_ROUND(i + 0, a, b, c, d, e, f, g, h); + SHA256_ROUND(i + 1, h, a, b, c, d, e, f, g); + SHA256_ROUND(i + 2, g, h, a, b, c, d, e, f); + SHA256_ROUND(i + 3, f, g, h, a, b, c, d, e); + SHA256_ROUND(i + 4, e, f, g, h, a, b, c, d); + SHA256_ROUND(i + 5, d, e, f, g, h, a, b, c); + SHA256_ROUND(i + 6, c, d, e, f, g, h, a, b); + SHA256_ROUND(i + 7, b, c, d, e, f, g, h, a); + } state[0] += a; state[1] += b; state[2] += c; state[3] += d; state[4] += e; state[5] += f; state[6] += g; state[7] += h; - - /* clear any sensitive info... */ - a = b = c = d = e = f = g = h = t1 = t2 = 0; - memzero_explicit(W, 64 * sizeof(u32)); } -int sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) +void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) { unsigned int partial, done; const u8 *src; + u32 W[64]; partial = sctx->count & 0x3f; sctx->count += len; @@ -224,26 +138,26 @@ int sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) } do { - sha256_transform(sctx->state, src); + sha256_transform(sctx->state, src, W); done += 64; src = data + done; } while (done + 63 < len); + memzero_explicit(W, sizeof(W)); + partial = 0; } memcpy(sctx->buf + partial, src, len - done); - - return 0; } EXPORT_SYMBOL(sha256_update); -int sha224_update(struct sha256_state *sctx, const u8 *data, unsigned int len) +void sha224_update(struct sha256_state *sctx, const u8 *data, unsigned int len) { - return sha256_update(sctx, data, len); + sha256_update(sctx, data, len); } EXPORT_SYMBOL(sha224_update); -static int __sha256_final(struct sha256_state *sctx, u8 *out, int digest_words) +static void __sha256_final(struct sha256_state *sctx, u8 *out, int digest_words) { __be32 *dst = (__be32 *)out; __be64 bits; @@ -267,21 +181,29 @@ static int __sha256_final(struct sha256_state *sctx, u8 *out, int digest_words) put_unaligned_be32(sctx->state[i], &dst[i]); /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; + memzero_explicit(sctx, sizeof(*sctx)); } -int sha256_final(struct sha256_state *sctx, u8 *out) +void sha256_final(struct sha256_state *sctx, u8 *out) { - return __sha256_final(sctx, out, 8); + __sha256_final(sctx, out, 8); } EXPORT_SYMBOL(sha256_final); -int sha224_final(struct sha256_state *sctx, u8 *out) +void sha224_final(struct sha256_state *sctx, u8 *out) { - return __sha256_final(sctx, out, 7); + __sha256_final(sctx, out, 7); } EXPORT_SYMBOL(sha224_final); +void sha256(const u8 *data, unsigned int len, u8 *out) +{ + struct sha256_state sctx; + + sha256_init(&sctx); + sha256_update(&sctx, data, len); + sha256_final(&sctx, out); +} +EXPORT_SYMBOL(sha256); + MODULE_LICENSE("GPL"); diff --git a/lib/crypto/sm4.c b/lib/crypto/sm4.c new file mode 100644 index 000000000000..284e62576d0c --- /dev/null +++ b/lib/crypto/sm4.c @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM4, as specified in + * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html + * + * Copyright (C) 2018 ARM Limited or its affiliates. + * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + */ + +#include <linux/module.h> +#include <asm/unaligned.h> +#include <crypto/sm4.h> + +static const u32 fk[4] = { + 0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc +}; + +static const u32 ____cacheline_aligned ck[32] = { + 0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269, + 0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9, + 0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249, + 0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9, + 0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229, + 0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299, + 0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209, + 0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279 +}; + +static const u8 ____cacheline_aligned sbox[256] = { + 0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7, + 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05, + 0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3, + 0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99, + 0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a, + 0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62, + 0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95, + 0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6, + 0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba, + 0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8, + 0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b, + 0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35, + 0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2, + 0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87, + 0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52, + 0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e, + 0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5, + 0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1, + 0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55, + 0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3, + 0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60, + 0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f, + 0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f, + 0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51, + 0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f, + 0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8, + 0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd, + 0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0, + 0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e, + 0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84, + 0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20, + 0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48 +}; + +static inline u32 sm4_t_non_lin_sub(u32 x) +{ + u32 out; + + out = (u32)sbox[x & 0xff]; + out |= (u32)sbox[(x >> 8) & 0xff] << 8; + out |= (u32)sbox[(x >> 16) & 0xff] << 16; + out |= (u32)sbox[(x >> 24) & 0xff] << 24; + + return out; +} + +static inline u32 sm4_key_lin_sub(u32 x) +{ + return x ^ rol32(x, 13) ^ rol32(x, 23); +} + +static inline u32 sm4_enc_lin_sub(u32 x) +{ + return x ^ rol32(x, 2) ^ rol32(x, 10) ^ rol32(x, 18) ^ rol32(x, 24); +} + +static inline u32 sm4_key_sub(u32 x) +{ + return sm4_key_lin_sub(sm4_t_non_lin_sub(x)); +} + +static inline u32 sm4_enc_sub(u32 x) +{ + return sm4_enc_lin_sub(sm4_t_non_lin_sub(x)); +} + +static inline u32 sm4_round(u32 x0, u32 x1, u32 x2, u32 x3, u32 rk) +{ + return x0 ^ sm4_enc_sub(x1 ^ x2 ^ x3 ^ rk); +} + + +/** + * sm4_expandkey - Expands the SM4 key as described in GB/T 32907-2016 + * @ctx: The location where the computed key will be stored. + * @in_key: The supplied key. + * @key_len: The length of the supplied key. + * + * Returns 0 on success. The function fails only if an invalid key size (or + * pointer) is supplied. + */ +int sm4_expandkey(struct sm4_ctx *ctx, const u8 *in_key, + unsigned int key_len) +{ + u32 rk[4]; + const u32 *key = (u32 *)in_key; + int i; + + if (key_len != SM4_KEY_SIZE) + return -EINVAL; + + rk[0] = get_unaligned_be32(&key[0]) ^ fk[0]; + rk[1] = get_unaligned_be32(&key[1]) ^ fk[1]; + rk[2] = get_unaligned_be32(&key[2]) ^ fk[2]; + rk[3] = get_unaligned_be32(&key[3]) ^ fk[3]; + + for (i = 0; i < 32; i += 4) { + rk[0] ^= sm4_key_sub(rk[1] ^ rk[2] ^ rk[3] ^ ck[i + 0]); + rk[1] ^= sm4_key_sub(rk[2] ^ rk[3] ^ rk[0] ^ ck[i + 1]); + rk[2] ^= sm4_key_sub(rk[3] ^ rk[0] ^ rk[1] ^ ck[i + 2]); + rk[3] ^= sm4_key_sub(rk[0] ^ rk[1] ^ rk[2] ^ ck[i + 3]); + + ctx->rkey_enc[i + 0] = rk[0]; + ctx->rkey_enc[i + 1] = rk[1]; + ctx->rkey_enc[i + 2] = rk[2]; + ctx->rkey_enc[i + 3] = rk[3]; + ctx->rkey_dec[31 - 0 - i] = rk[0]; + ctx->rkey_dec[31 - 1 - i] = rk[1]; + ctx->rkey_dec[31 - 2 - i] = rk[2]; + ctx->rkey_dec[31 - 3 - i] = rk[3]; + } + + return 0; +} +EXPORT_SYMBOL_GPL(sm4_expandkey); + +/** + * sm4_crypt_block - Encrypt or decrypt a single SM4 block + * @rk: The rkey_enc for encrypt or rkey_dec for decrypt + * @out: Buffer to store output data + * @in: Buffer containing the input data + */ +void sm4_crypt_block(const u32 *rk, u8 *out, const u8 *in) +{ + u32 x[4], i; + + x[0] = get_unaligned_be32(in + 0 * 4); + x[1] = get_unaligned_be32(in + 1 * 4); + x[2] = get_unaligned_be32(in + 2 * 4); + x[3] = get_unaligned_be32(in + 3 * 4); + + for (i = 0; i < 32; i += 4) { + x[0] = sm4_round(x[0], x[1], x[2], x[3], rk[i + 0]); + x[1] = sm4_round(x[1], x[2], x[3], x[0], rk[i + 1]); + x[2] = sm4_round(x[2], x[3], x[0], x[1], rk[i + 2]); + x[3] = sm4_round(x[3], x[0], x[1], x[2], rk[i + 3]); + } + + put_unaligned_be32(x[3 - 0], out + 0 * 4); + put_unaligned_be32(x[3 - 1], out + 1 * 4); + put_unaligned_be32(x[3 - 2], out + 2 * 4); + put_unaligned_be32(x[3 - 3], out + 3 * 4); +} +EXPORT_SYMBOL_GPL(sm4_crypt_block); + +MODULE_DESCRIPTION("Generic SM4 library"); +MODULE_LICENSE("GPL v2"); diff --git a/lib/debug_info.c b/lib/debug_info.c index 36daf753293c..cc4723c74af5 100644 --- a/lib/debug_info.c +++ b/lib/debug_info.c @@ -5,8 +5,6 @@ * CONFIG_DEBUG_INFO_REDUCED. Please do not add actual code. However, * adding appropriate #includes is fine. */ -#include <stdarg.h> - #include <linux/cred.h> #include <linux/crypto.h> #include <linux/dcache.h> @@ -22,6 +20,7 @@ #include <linux/net.h> #include <linux/sched.h> #include <linux/slab.h> +#include <linux/stdarg.h> #include <linux/types.h> #include <net/addrconf.h> #include <net/sock.h> diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 61261195f5b6..6946f8e204e3 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -19,6 +19,7 @@ #include <linux/slab.h> #include <linux/hash.h> #include <linux/kmemleak.h> +#include <linux/cpu.h> #define ODEBUG_HASH_BITS 14 #define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS) @@ -90,7 +91,7 @@ static int debug_objects_pool_size __read_mostly = ODEBUG_POOL_SIZE; static int debug_objects_pool_min_level __read_mostly = ODEBUG_POOL_MIN_LEVEL; -static struct debug_obj_descr *descr_test __read_mostly; +static const struct debug_obj_descr *descr_test __read_mostly; static struct kmem_cache *obj_cache __read_mostly; /* @@ -132,14 +133,18 @@ static void fill_pool(void) struct debug_obj *obj; unsigned long flags; - if (likely(obj_pool_free >= debug_objects_pool_min_level)) + if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level)) return; /* * Reuse objs from the global free list; they will be reinitialized * when allocating. + * + * Both obj_nr_tofree and obj_pool_free are checked locklessly; the + * READ_ONCE()s pair with the WRITE_ONCE()s in pool_lock critical + * sections. */ - while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) { + while (READ_ONCE(obj_nr_tofree) && (READ_ONCE(obj_pool_free) < obj_pool_min_free)) { raw_spin_lock_irqsave(&pool_lock, flags); /* * Recheck with the lock held as the worker thread might have @@ -148,9 +153,9 @@ static void fill_pool(void) while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) { obj = hlist_entry(obj_to_free.first, typeof(*obj), node); hlist_del(&obj->node); - obj_nr_tofree--; + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); hlist_add_head(&obj->node, &obj_pool); - obj_pool_free++; + WRITE_ONCE(obj_pool_free, obj_pool_free + 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); } @@ -158,7 +163,7 @@ static void fill_pool(void) if (unlikely(!obj_cache)) return; - while (obj_pool_free < debug_objects_pool_min_level) { + while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) { struct debug_obj *new[ODEBUG_BATCH_SIZE]; int cnt; @@ -174,7 +179,7 @@ static void fill_pool(void) while (cnt) { hlist_add_head(&new[--cnt]->node, &obj_pool); debug_objects_allocated++; - obj_pool_free++; + WRITE_ONCE(obj_pool_free, obj_pool_free + 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); } @@ -219,7 +224,7 @@ static struct debug_obj *__alloc_object(struct hlist_head *list) * Must be called with interrupts disabled. */ static struct debug_obj * -alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) +alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr) { struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool); struct debug_obj *obj; @@ -236,7 +241,7 @@ alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) obj = __alloc_object(&obj_pool); if (obj) { obj_pool_used++; - obj_pool_free--; + WRITE_ONCE(obj_pool_free, obj_pool_free - 1); /* * Looking ahead, allocate one batch of debug objects and @@ -255,7 +260,7 @@ alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) &percpu_pool->free_objs); percpu_pool->obj_free++; obj_pool_used++; - obj_pool_free--; + WRITE_ONCE(obj_pool_free, obj_pool_free - 1); } } @@ -309,8 +314,8 @@ static void free_obj_work(struct work_struct *work) obj = hlist_entry(obj_to_free.first, typeof(*obj), node); hlist_del(&obj->node); hlist_add_head(&obj->node, &obj_pool); - obj_pool_free++; - obj_nr_tofree--; + WRITE_ONCE(obj_pool_free, obj_pool_free + 1); + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); return; @@ -324,7 +329,7 @@ free_objs: if (obj_nr_tofree) { hlist_move_list(&obj_to_free, &tofree); debug_objects_freed += obj_nr_tofree; - obj_nr_tofree = 0; + WRITE_ONCE(obj_nr_tofree, 0); } raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -375,10 +380,10 @@ free_to_obj_pool: obj_pool_used--; if (work) { - obj_nr_tofree++; + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1); hlist_add_head(&obj->node, &obj_to_free); if (lookahead_count) { - obj_nr_tofree += lookahead_count; + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count); obj_pool_used -= lookahead_count; while (lookahead_count) { hlist_add_head(&objs[--lookahead_count]->node, @@ -396,15 +401,15 @@ free_to_obj_pool: for (i = 0; i < ODEBUG_BATCH_SIZE; i++) { obj = __alloc_object(&obj_pool); hlist_add_head(&obj->node, &obj_to_free); - obj_pool_free--; - obj_nr_tofree++; + WRITE_ONCE(obj_pool_free, obj_pool_free - 1); + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1); } } } else { - obj_pool_free++; + WRITE_ONCE(obj_pool_free, obj_pool_free + 1); hlist_add_head(&obj->node, &obj_pool); if (lookahead_count) { - obj_pool_free += lookahead_count; + WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count); obj_pool_used -= lookahead_count; while (lookahead_count) { hlist_add_head(&objs[--lookahead_count]->node, @@ -423,12 +428,31 @@ free_to_obj_pool: static void free_object(struct debug_obj *obj) { __free_object(obj); - if (!obj_freeing && obj_nr_tofree) { + if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) { WRITE_ONCE(obj_freeing, true); schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY); } } +#ifdef CONFIG_HOTPLUG_CPU +static int object_cpu_offline(unsigned int cpu) +{ + struct debug_percpu_free *percpu_pool; + struct hlist_node *tmp; + struct debug_obj *obj; + + /* Remote access is safe as the CPU is dead already */ + percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu); + hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) { + hlist_del(&obj->node); + kmem_cache_free(obj_cache, obj); + } + percpu_pool->obj_free = 0; + + return 0; +} +#endif + /* * We run out of memory. That means we probably have tons of objects * allocated. @@ -471,7 +495,7 @@ static struct debug_bucket *get_bucket(unsigned long addr) static void debug_print_object(struct debug_obj *obj, char *msg) { - struct debug_obj_descr *descr = obj->descr; + const struct debug_obj_descr *descr = obj->descr; static int limit; if (limit < 5 && descr != descr_test) { @@ -525,7 +549,7 @@ static void debug_object_is_on_stack(void *addr, int onstack) } static void -__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack) +__debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack) { enum debug_obj_state state; bool check_stack = false; @@ -533,7 +557,12 @@ __debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack) struct debug_obj *obj; unsigned long flags; - fill_pool(); + /* + * On RT enabled kernels the pool refill must happen in preemptible + * context: + */ + if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible()) + fill_pool(); db = get_bucket((unsigned long) addr); @@ -583,7 +612,7 @@ __debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack) * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_init(void *addr, struct debug_obj_descr *descr) +void debug_object_init(void *addr, const struct debug_obj_descr *descr) { if (!debug_objects_enabled) return; @@ -598,7 +627,7 @@ EXPORT_SYMBOL_GPL(debug_object_init); * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr) +void debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr) { if (!debug_objects_enabled) return; @@ -613,7 +642,7 @@ EXPORT_SYMBOL_GPL(debug_object_init_on_stack); * @descr: pointer to an object specific debug description structure * Returns 0 for success, -EINVAL for check failed. */ -int debug_object_activate(void *addr, struct debug_obj_descr *descr) +int debug_object_activate(void *addr, const struct debug_obj_descr *descr) { enum debug_obj_state state; struct debug_bucket *db; @@ -691,7 +720,7 @@ EXPORT_SYMBOL_GPL(debug_object_activate); * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_deactivate(void *addr, struct debug_obj_descr *descr) +void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr) { struct debug_bucket *db; struct debug_obj *obj; @@ -743,7 +772,7 @@ EXPORT_SYMBOL_GPL(debug_object_deactivate); * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_destroy(void *addr, struct debug_obj_descr *descr) +void debug_object_destroy(void *addr, const struct debug_obj_descr *descr) { enum debug_obj_state state; struct debug_bucket *db; @@ -793,7 +822,7 @@ EXPORT_SYMBOL_GPL(debug_object_destroy); * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_free(void *addr, struct debug_obj_descr *descr) +void debug_object_free(void *addr, const struct debug_obj_descr *descr) { enum debug_obj_state state; struct debug_bucket *db; @@ -834,7 +863,7 @@ EXPORT_SYMBOL_GPL(debug_object_free); * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_assert_init(void *addr, struct debug_obj_descr *descr) +void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr) { struct debug_bucket *db; struct debug_obj *obj; @@ -882,7 +911,7 @@ EXPORT_SYMBOL_GPL(debug_object_assert_init); * @next: state to move to if expected state is found */ void -debug_object_active_state(void *addr, struct debug_obj_descr *descr, +debug_object_active_state(void *addr, const struct debug_obj_descr *descr, unsigned int expect, unsigned int next) { struct debug_bucket *db; @@ -930,7 +959,7 @@ EXPORT_SYMBOL_GPL(debug_object_active_state); static void __debug_check_no_obj_freed(const void *address, unsigned long size) { unsigned long flags, oaddr, saddr, eaddr, paddr, chunks; - struct debug_obj_descr *descr; + const struct debug_obj_descr *descr; enum debug_obj_state state; struct debug_bucket *db; struct hlist_node *tmp; @@ -982,7 +1011,7 @@ repeat: debug_objects_maxchecked = objs_checked; /* Schedule work to actually kmem_cache_free() objects */ - if (!obj_freeing && obj_nr_tofree) { + if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) { WRITE_ONCE(obj_freeing, true); schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY); } @@ -1008,28 +1037,17 @@ static int debug_stats_show(struct seq_file *m, void *v) seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked); seq_printf(m, "warnings :%d\n", debug_objects_warnings); seq_printf(m, "fixups :%d\n", debug_objects_fixups); - seq_printf(m, "pool_free :%d\n", obj_pool_free + obj_percpu_free); + seq_printf(m, "pool_free :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free); seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free); seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free); seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free); seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used); - seq_printf(m, "on_free_list :%d\n", obj_nr_tofree); + seq_printf(m, "on_free_list :%d\n", READ_ONCE(obj_nr_tofree)); seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated); seq_printf(m, "objs_freed :%d\n", debug_objects_freed); return 0; } - -static int debug_stats_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, debug_stats_show, NULL); -} - -static const struct file_operations debug_stats_fops = { - .open = debug_stats_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(debug_stats); static int __init debug_objects_init_debugfs(void) { @@ -1059,7 +1077,7 @@ struct self_test { unsigned long dummy2[3]; }; -static __initdata struct debug_obj_descr descr_type_test; +static __initconst const struct debug_obj_descr descr_type_test; static bool __init is_static_object(void *addr) { @@ -1184,7 +1202,7 @@ out: return res; } -static __initdata struct debug_obj_descr descr_type_test = { +static __initconst const struct debug_obj_descr descr_type_test = { .name = "selftest", .is_static_object = is_static_object, .fixup_init = fixup_init, @@ -1374,6 +1392,11 @@ void __init debug_objects_mem_init(void) } else debug_objects_selftest(); +#ifdef CONFIG_HOTPLUG_CPU + cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL, + object_cpu_offline); +#endif + /* * Increase the thresholds for allocating and freeing objects * according to the number of possible CPUs available in the system. diff --git a/lib/decompress.c b/lib/decompress.c index 857ab1af1ef3..ab3fc90ffc64 100644 --- a/lib/decompress.c +++ b/lib/decompress.c @@ -13,6 +13,7 @@ #include <linux/decompress/inflate.h> #include <linux/decompress/unlzo.h> #include <linux/decompress/unlz4.h> +#include <linux/decompress/unzstd.h> #include <linux/types.h> #include <linux/string.h> @@ -37,6 +38,9 @@ #ifndef CONFIG_DECOMPRESS_LZ4 # define unlz4 NULL #endif +#ifndef CONFIG_DECOMPRESS_ZSTD +# define unzstd NULL +#endif struct compress_format { unsigned char magic[2]; @@ -52,6 +56,7 @@ static const struct compress_format compressed_formats[] __initconst = { { {0xfd, 0x37}, "xz", unxz }, { {0x89, 0x4c}, "lzo", unlzo }, { {0x02, 0x21}, "lz4", unlz4 }, + { {0x28, 0xb5}, "zstd", unzstd }, { {0, 0}, NULL, NULL } }; diff --git a/lib/decompress_bunzip2.c b/lib/decompress_bunzip2.c index 7c4932eed748..3518e7394eca 100644 --- a/lib/decompress_bunzip2.c +++ b/lib/decompress_bunzip2.c @@ -34,7 +34,7 @@ Phone (337) 232-1234 or 1-800-738-2226 Fax (337) 232-1297 - http://www.hospiceacadiana.com/ + https://www.hospiceacadiana.com/ Manuel */ @@ -80,7 +80,7 @@ /* This is what we know about each Huffman coding group */ struct group_data { - /* We have an extra slot at the end of limit[] for a sentinal value. */ + /* We have an extra slot at the end of limit[] for a sentinel value. */ int limit[MAX_HUFCODE_BITS+1]; int base[MAX_HUFCODE_BITS]; int permute[MAX_SYMBOLS]; @@ -337,7 +337,7 @@ static int INIT get_next_block(struct bunzip_data *bd) pp <<= 1; base[i+1] = pp-(t += temp[i]); } - limit[maxLen+1] = INT_MAX; /* Sentinal value for + limit[maxLen+1] = INT_MAX; /* Sentinel value for * reading next sym. */ limit[maxLen] = pp+temp[maxLen]-1; base[minLen] = 0; @@ -385,12 +385,12 @@ static int INIT get_next_block(struct bunzip_data *bd) bd->inbufBits = (bd->inbufBits << 8)|bd->inbuf[bd->inbufPos++]; bd->inbufBitCount += 8; - }; + } bd->inbufBitCount -= hufGroup->maxLen; j = (bd->inbufBits >> bd->inbufBitCount)& ((1 << hufGroup->maxLen)-1); got_huff_bits: - /* Figure how how many bits are in next symbol and + /* Figure how many bits are in next symbol and * unget extras */ i = hufGroup->minLen; while (j > limit[i]) diff --git a/lib/decompress_inflate.c b/lib/decompress_inflate.c index 63b4b7eee138..6130c42b8e59 100644 --- a/lib/decompress_inflate.c +++ b/lib/decompress_inflate.c @@ -10,6 +10,10 @@ #include "zlib_inflate/inftrees.c" #include "zlib_inflate/inffast.c" #include "zlib_inflate/inflate.c" +#ifdef CONFIG_ZLIB_DFLTCC +#include "zlib_dfltcc/dfltcc.c" +#include "zlib_dfltcc/dfltcc_inflate.c" +#endif #else /* STATIC */ /* initramfs et al: linked */ @@ -76,7 +80,12 @@ STATIC int INIT __gunzip(unsigned char *buf, long len, } strm->workspace = malloc(flush ? zlib_inflate_workspacesize() : +#ifdef CONFIG_ZLIB_DFLTCC + /* Always allocate the full workspace for DFLTCC */ + zlib_inflate_workspacesize()); +#else sizeof(struct inflate_state)); +#endif if (strm->workspace == NULL) { error("Out of memory while allocating workspace"); goto gunzip_nomem4; @@ -123,10 +132,14 @@ STATIC int INIT __gunzip(unsigned char *buf, long len, rc = zlib_inflateInit2(strm, -MAX_WBITS); +#ifdef CONFIG_ZLIB_DFLTCC + /* Always keep the window for DFLTCC */ +#else if (!flush) { WS(strm)->inflate_state.wsize = 0; WS(strm)->inflate_state.window = NULL; } +#endif while (rc == Z_OK) { if (strm->avail_in == 0) { diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c index c0cfcfd486be..e6327391b6b6 100644 --- a/lib/decompress_unlz4.c +++ b/lib/decompress_unlz4.c @@ -112,6 +112,9 @@ STATIC inline int INIT unlz4(u8 *input, long in_len, error("data corrupted"); goto exit_2; } + } else if (size < 4) { + /* empty or end-of-file */ + goto exit_3; } chunksize = get_unaligned_le32(inp); @@ -125,6 +128,10 @@ STATIC inline int INIT unlz4(u8 *input, long in_len, continue; } + if (!fill && chunksize == 0) { + /* empty or end-of-file */ + goto exit_3; + } if (posp) *posp += 4; @@ -184,6 +191,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len, } } +exit_3: ret = 0; exit_2: if (!input) diff --git a/lib/decompress_unlzma.c b/lib/decompress_unlzma.c index ed7a1fd819f2..20a858031f12 100644 --- a/lib/decompress_unlzma.c +++ b/lib/decompress_unlzma.c @@ -8,7 +8,7 @@ *implementation for lzma. *Copyright (C) 2006 Aurelien Jacobs < aurel@gnuage.org > * - *Based on LzmaDecode.c from the LZMA SDK 4.22 (http://www.7-zip.org/) + *Based on LzmaDecode.c from the LZMA SDK 4.22 (https://www.7-zip.org/) *Copyright (C) 1999-2005 Igor Pavlov * *Copyrights of the parts, see headers below. @@ -56,7 +56,7 @@ static long long INIT read_int(unsigned char *ptr, int size) /* Small range coder implementation for lzma. *Copyright (C) 2006 Aurelien Jacobs < aurel@gnuage.org > * - *Based on LzmaDecode.c from the LZMA SDK 4.22 (http://www.7-zip.org/) + *Based on LzmaDecode.c from the LZMA SDK 4.22 (https://www.7-zip.org/) *Copyright (c) 1999-2005 Igor Pavlov */ @@ -213,7 +213,7 @@ rc_bit_tree_decode(struct rc *rc, uint16_t *p, int num_levels, int *symbol) * Small lzma deflate implementation. * Copyright (C) 2006 Aurelien Jacobs < aurel@gnuage.org > * - * Based on LzmaDecode.c from the LZMA SDK 4.22 (http://www.7-zip.org/) + * Based on LzmaDecode.c from the LZMA SDK 4.22 (https://www.7-zip.org/) * Copyright (C) 1999-2005 Igor Pavlov */ @@ -391,7 +391,7 @@ static inline int INIT process_bit0(struct writer *wr, struct rc *rc, static inline int INIT process_bit1(struct writer *wr, struct rc *rc, struct cstate *cst, uint16_t *p, int pos_state, uint16_t *prob) { - int offset; + int offset; uint16_t *prob_len; int num_bits; int len; diff --git a/lib/decompress_unlzo.c b/lib/decompress_unlzo.c index 1f439a622076..64c1358500ce 100644 --- a/lib/decompress_unlzo.c +++ b/lib/decompress_unlzo.c @@ -43,7 +43,6 @@ STATIC inline long INIT parse_header(u8 *input, long *skip, long in_len) int l; u8 *parse = input; u8 *end = input + in_len; - u8 level = 0; u16 version; /* @@ -65,7 +64,7 @@ STATIC inline long INIT parse_header(u8 *input, long *skip, long in_len) version = get_unaligned_be16(parse); parse += 7; if (version >= 0x0940) - level = *parse++; + parse++; if (get_unaligned_be32(parse) & HEADER_HAS_FILTER) parse += 8; /* flags + filter info */ else diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c index 25d59a95bd66..9f4262ee33a5 100644 --- a/lib/decompress_unxz.c +++ b/lib/decompress_unxz.c @@ -20,10 +20,10 @@ * * The worst case for in-place decompression is that the beginning of * the file is compressed extremely well, and the rest of the file is - * uncompressible. Thus, we must look for worst-case expansion when the - * compressor is encoding uncompressible data. + * incompressible. Thus, we must look for worst-case expansion when the + * compressor is encoding incompressible data. * - * The structure of the .xz file in case of a compresed kernel is as follows. + * The structure of the .xz file in case of a compressed kernel is as follows. * Sizes (as bytes) of the fields are in parenthesis. * * Stream Header (12) @@ -58,7 +58,7 @@ * uncompressed size of the payload is in practice never less than the * payload size itself. The LZMA2 format would allow uncompressed size * to be less than the payload size, but no sane compressor creates such - * files. LZMA2 supports storing uncompressible data in uncompressed form, + * files. LZMA2 supports storing incompressible data in uncompressed form, * so there's never a need to create payloads whose uncompressed size is * smaller than the compressed size. * @@ -167,8 +167,8 @@ * memeq and memzero are not used much and any remotely sane implementation * is fast enough. memcpy/memmove speed matters in multi-call mode, but * the kernel image is decompressed in single-call mode, in which only - * memcpy speed can matter and only if there is a lot of uncompressible data - * (LZMA2 stores uncompressible chunks in uncompressed form). Thus, the + * memmove speed can matter and only if there is a lot of incompressible data + * (LZMA2 stores incompressible chunks in uncompressed form). Thus, the * functions below should just be kept small; it's probably not worth * optimizing for speed. */ diff --git a/lib/decompress_unzstd.c b/lib/decompress_unzstd.c new file mode 100644 index 000000000000..6b629ab31c1e --- /dev/null +++ b/lib/decompress_unzstd.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Important notes about in-place decompression + * + * At least on x86, the kernel is decompressed in place: the compressed data + * is placed to the end of the output buffer, and the decompressor overwrites + * most of the compressed data. There must be enough safety margin to + * guarantee that the write position is always behind the read position. + * + * The safety margin for ZSTD with a 128 KB block size is calculated below. + * Note that the margin with ZSTD is bigger than with GZIP or XZ! + * + * The worst case for in-place decompression is that the beginning of + * the file is compressed extremely well, and the rest of the file is + * uncompressible. Thus, we must look for worst-case expansion when the + * compressor is encoding uncompressible data. + * + * The structure of the .zst file in case of a compressed kernel is as follows. + * Maximum sizes (as bytes) of the fields are in parenthesis. + * + * Frame Header: (18) + * Blocks: (N) + * Checksum: (4) + * + * The frame header and checksum overhead is at most 22 bytes. + * + * ZSTD stores the data in blocks. Each block has a header whose size is + * a 3 bytes. After the block header, there is up to 128 KB of payload. + * The maximum uncompressed size of the payload is 128 KB. The minimum + * uncompressed size of the payload is never less than the payload size + * (excluding the block header). + * + * The assumption, that the uncompressed size of the payload is never + * smaller than the payload itself, is valid only when talking about + * the payload as a whole. It is possible that the payload has parts where + * the decompressor consumes more input than it produces output. Calculating + * the worst case for this would be tricky. Instead of trying to do that, + * let's simply make sure that the decompressor never overwrites any bytes + * of the payload which it is currently reading. + * + * Now we have enough information to calculate the safety margin. We need + * - 22 bytes for the .zst file format headers; + * - 3 bytes per every 128 KiB of uncompressed size (one block header per + * block); and + * - 128 KiB (biggest possible zstd block size) to make sure that the + * decompressor never overwrites anything from the block it is currently + * reading. + * + * We get the following formula: + * + * safety_margin = 22 + uncompressed_size * 3 / 131072 + 131072 + * <= 22 + (uncompressed_size >> 15) + 131072 + */ + +/* + * Preboot environments #include "path/to/decompress_unzstd.c". + * All of the source files we depend on must be #included. + * zstd's only source dependency is xxhash, which has no source + * dependencies. + * + * When UNZSTD_PREBOOT is defined we declare __decompress(), which is + * used for kernel decompression, instead of unzstd(). + * + * Define __DISABLE_EXPORTS in preboot environments to prevent symbols + * from xxhash and zstd from being exported by the EXPORT_SYMBOL macro. + */ +#ifdef STATIC +# define UNZSTD_PREBOOT +# include "xxhash.c" +# include "zstd/entropy_common.c" +# include "zstd/fse_decompress.c" +# include "zstd/huf_decompress.c" +# include "zstd/zstd_common.c" +# include "zstd/decompress.c" +#endif + +#include <linux/decompress/mm.h> +#include <linux/kernel.h> +#include <linux/zstd.h> + +/* 128MB is the maximum window size supported by zstd. */ +#define ZSTD_WINDOWSIZE_MAX (1 << ZSTD_WINDOWLOG_MAX) +/* + * Size of the input and output buffers in multi-call mode. + * Pick a larger size because it isn't used during kernel decompression, + * since that is single pass, and we have to allocate a large buffer for + * zstd's window anyway. The larger size speeds up initramfs decompression. + */ +#define ZSTD_IOBUF_SIZE (1 << 17) + +static int INIT handle_zstd_error(size_t ret, void (*error)(char *x)) +{ + const int err = ZSTD_getErrorCode(ret); + + if (!ZSTD_isError(ret)) + return 0; + + switch (err) { + case ZSTD_error_memory_allocation: + error("ZSTD decompressor ran out of memory"); + break; + case ZSTD_error_prefix_unknown: + error("Input is not in the ZSTD format (wrong magic bytes)"); + break; + case ZSTD_error_dstSize_tooSmall: + case ZSTD_error_corruption_detected: + case ZSTD_error_checksum_wrong: + error("ZSTD-compressed data is corrupt"); + break; + default: + error("ZSTD-compressed data is probably corrupt"); + break; + } + return -1; +} + +/* + * Handle the case where we have the entire input and output in one segment. + * We can allocate less memory (no circular buffer for the sliding window), + * and avoid some memcpy() calls. + */ +static int INIT decompress_single(const u8 *in_buf, long in_len, u8 *out_buf, + long out_len, long *in_pos, + void (*error)(char *x)) +{ + const size_t wksp_size = ZSTD_DCtxWorkspaceBound(); + void *wksp = large_malloc(wksp_size); + ZSTD_DCtx *dctx = ZSTD_initDCtx(wksp, wksp_size); + int err; + size_t ret; + + if (dctx == NULL) { + error("Out of memory while allocating ZSTD_DCtx"); + err = -1; + goto out; + } + /* + * Find out how large the frame actually is, there may be junk at + * the end of the frame that ZSTD_decompressDCtx() can't handle. + */ + ret = ZSTD_findFrameCompressedSize(in_buf, in_len); + err = handle_zstd_error(ret, error); + if (err) + goto out; + in_len = (long)ret; + + ret = ZSTD_decompressDCtx(dctx, out_buf, out_len, in_buf, in_len); + err = handle_zstd_error(ret, error); + if (err) + goto out; + + if (in_pos != NULL) + *in_pos = in_len; + + err = 0; +out: + if (wksp != NULL) + large_free(wksp); + return err; +} + +static int INIT __unzstd(unsigned char *in_buf, long in_len, + long (*fill)(void*, unsigned long), + long (*flush)(void*, unsigned long), + unsigned char *out_buf, long out_len, + long *in_pos, + void (*error)(char *x)) +{ + ZSTD_inBuffer in; + ZSTD_outBuffer out; + ZSTD_frameParams params; + void *in_allocated = NULL; + void *out_allocated = NULL; + void *wksp = NULL; + size_t wksp_size; + ZSTD_DStream *dstream; + int err; + size_t ret; + + /* + * ZSTD decompression code won't be happy if the buffer size is so big + * that its end address overflows. When the size is not provided, make + * it as big as possible without having the end address overflow. + */ + if (out_len == 0) + out_len = UINTPTR_MAX - (uintptr_t)out_buf; + + if (fill == NULL && flush == NULL) + /* + * We can decompress faster and with less memory when we have a + * single chunk. + */ + return decompress_single(in_buf, in_len, out_buf, out_len, + in_pos, error); + + /* + * If in_buf is not provided, we must be using fill(), so allocate + * a large enough buffer. If it is provided, it must be at least + * ZSTD_IOBUF_SIZE large. + */ + if (in_buf == NULL) { + in_allocated = large_malloc(ZSTD_IOBUF_SIZE); + if (in_allocated == NULL) { + error("Out of memory while allocating input buffer"); + err = -1; + goto out; + } + in_buf = in_allocated; + in_len = 0; + } + /* Read the first chunk, since we need to decode the frame header. */ + if (fill != NULL) + in_len = fill(in_buf, ZSTD_IOBUF_SIZE); + if (in_len < 0) { + error("ZSTD-compressed data is truncated"); + err = -1; + goto out; + } + /* Set the first non-empty input buffer. */ + in.src = in_buf; + in.pos = 0; + in.size = in_len; + /* Allocate the output buffer if we are using flush(). */ + if (flush != NULL) { + out_allocated = large_malloc(ZSTD_IOBUF_SIZE); + if (out_allocated == NULL) { + error("Out of memory while allocating output buffer"); + err = -1; + goto out; + } + out_buf = out_allocated; + out_len = ZSTD_IOBUF_SIZE; + } + /* Set the output buffer. */ + out.dst = out_buf; + out.pos = 0; + out.size = out_len; + + /* + * We need to know the window size to allocate the ZSTD_DStream. + * Since we are streaming, we need to allocate a buffer for the sliding + * window. The window size varies from 1 KB to ZSTD_WINDOWSIZE_MAX + * (8 MB), so it is important to use the actual value so as not to + * waste memory when it is smaller. + */ + ret = ZSTD_getFrameParams(¶ms, in.src, in.size); + err = handle_zstd_error(ret, error); + if (err) + goto out; + if (ret != 0) { + error("ZSTD-compressed data has an incomplete frame header"); + err = -1; + goto out; + } + if (params.windowSize > ZSTD_WINDOWSIZE_MAX) { + error("ZSTD-compressed data has too large a window size"); + err = -1; + goto out; + } + + /* + * Allocate the ZSTD_DStream now that we know how much memory is + * required. + */ + wksp_size = ZSTD_DStreamWorkspaceBound(params.windowSize); + wksp = large_malloc(wksp_size); + dstream = ZSTD_initDStream(params.windowSize, wksp, wksp_size); + if (dstream == NULL) { + error("Out of memory while allocating ZSTD_DStream"); + err = -1; + goto out; + } + + /* + * Decompression loop: + * Read more data if necessary (error if no more data can be read). + * Call the decompression function, which returns 0 when finished. + * Flush any data produced if using flush(). + */ + if (in_pos != NULL) + *in_pos = 0; + do { + /* + * If we need to reload data, either we have fill() and can + * try to get more data, or we don't and the input is truncated. + */ + if (in.pos == in.size) { + if (in_pos != NULL) + *in_pos += in.pos; + in_len = fill ? fill(in_buf, ZSTD_IOBUF_SIZE) : -1; + if (in_len < 0) { + error("ZSTD-compressed data is truncated"); + err = -1; + goto out; + } + in.pos = 0; + in.size = in_len; + } + /* Returns zero when the frame is complete. */ + ret = ZSTD_decompressStream(dstream, &out, &in); + err = handle_zstd_error(ret, error); + if (err) + goto out; + /* Flush all of the data produced if using flush(). */ + if (flush != NULL && out.pos > 0) { + if (out.pos != flush(out.dst, out.pos)) { + error("Failed to flush()"); + err = -1; + goto out; + } + out.pos = 0; + } + } while (ret != 0); + + if (in_pos != NULL) + *in_pos += in.pos; + + err = 0; +out: + if (in_allocated != NULL) + large_free(in_allocated); + if (out_allocated != NULL) + large_free(out_allocated); + if (wksp != NULL) + large_free(wksp); + return err; +} + +#ifndef UNZSTD_PREBOOT +STATIC int INIT unzstd(unsigned char *buf, long len, + long (*fill)(void*, unsigned long), + long (*flush)(void*, unsigned long), + unsigned char *out_buf, + long *pos, + void (*error)(char *x)) +{ + return __unzstd(buf, len, fill, flush, out_buf, 0, pos, error); +} +#else +STATIC int INIT __decompress(unsigned char *buf, long len, + long (*fill)(void*, unsigned long), + long (*flush)(void*, unsigned long), + unsigned char *out_buf, long out_len, + long *pos, + void (*error)(char *x)) +{ + return __unzstd(buf, len, fill, flush, out_buf, out_len, pos, error); +} +#endif diff --git a/lib/devmem_is_allowed.c b/lib/devmem_is_allowed.c new file mode 100644 index 000000000000..60be9e24bd57 --- /dev/null +++ b/lib/devmem_is_allowed.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * A generic version of devmem_is_allowed. + * + * Based on arch/arm64/mm/mmap.c + * + * Copyright (C) 2020 Google, Inc. + * Copyright (C) 2012 ARM Ltd. + */ + +#include <linux/mm.h> +#include <linux/ioport.h> + +/* + * devmem_is_allowed() checks to see if /dev/mem access to a certain address + * is valid. The argument is a physical page number. We mimic x86 here by + * disallowing access to system RAM as well as device-exclusive MMIO regions. + * This effectively disable read()/write() on /dev/mem. + */ +int devmem_is_allowed(unsigned long pfn) +{ + if (iomem_is_exclusive(PFN_PHYS(pfn))) + return 0; + if (!page_is_ram(pfn)) + return 1; + return 0; +} diff --git a/lib/devres.c b/lib/devres.c index f56070cf970b..14664bbb4875 100644 --- a/lib/devres.c +++ b/lib/devres.c @@ -8,9 +8,9 @@ enum devm_ioremap_type { DEVM_IOREMAP = 0, - DEVM_IOREMAP_NC, DEVM_IOREMAP_UC, DEVM_IOREMAP_WC, + DEVM_IOREMAP_NP, }; void devm_ioremap_release(struct device *dev, void *res) @@ -37,15 +37,15 @@ static void __iomem *__devm_ioremap(struct device *dev, resource_size_t offset, case DEVM_IOREMAP: addr = ioremap(offset, size); break; - case DEVM_IOREMAP_NC: - addr = ioremap_nocache(offset, size); - break; case DEVM_IOREMAP_UC: addr = ioremap_uc(offset, size); break; case DEVM_IOREMAP_WC: addr = ioremap_wc(offset, size); break; + case DEVM_IOREMAP_NP: + addr = ioremap_np(offset, size); + break; } if (addr) { @@ -88,35 +88,34 @@ void __iomem *devm_ioremap_uc(struct device *dev, resource_size_t offset, EXPORT_SYMBOL_GPL(devm_ioremap_uc); /** - * devm_ioremap_nocache - Managed ioremap_nocache() + * devm_ioremap_wc - Managed ioremap_wc() * @dev: Generic device to remap IO address for * @offset: Resource address to map * @size: Size of map * - * Managed ioremap_nocache(). Map is automatically unmapped on driver - * detach. + * Managed ioremap_wc(). Map is automatically unmapped on driver detach. */ -void __iomem *devm_ioremap_nocache(struct device *dev, resource_size_t offset, - resource_size_t size) +void __iomem *devm_ioremap_wc(struct device *dev, resource_size_t offset, + resource_size_t size) { - return __devm_ioremap(dev, offset, size, DEVM_IOREMAP_NC); + return __devm_ioremap(dev, offset, size, DEVM_IOREMAP_WC); } -EXPORT_SYMBOL(devm_ioremap_nocache); +EXPORT_SYMBOL(devm_ioremap_wc); /** - * devm_ioremap_wc - Managed ioremap_wc() + * devm_ioremap_np - Managed ioremap_np() * @dev: Generic device to remap IO address for * @offset: Resource address to map * @size: Size of map * - * Managed ioremap_wc(). Map is automatically unmapped on driver detach. + * Managed ioremap_np(). Map is automatically unmapped on driver detach. */ -void __iomem *devm_ioremap_wc(struct device *dev, resource_size_t offset, +void __iomem *devm_ioremap_np(struct device *dev, resource_size_t offset, resource_size_t size) { - return __devm_ioremap(dev, offset, size, DEVM_IOREMAP_WC); + return __devm_ioremap(dev, offset, size, DEVM_IOREMAP_NP); } -EXPORT_SYMBOL(devm_ioremap_wc); +EXPORT_SYMBOL(devm_ioremap_np); /** * devm_iounmap - Managed iounmap() @@ -139,6 +138,7 @@ __devm_ioremap_resource(struct device *dev, const struct resource *res, { resource_size_t size; void __iomem *dest_ptr; + char *pretty_name; BUG_ON(!dev); @@ -147,9 +147,22 @@ __devm_ioremap_resource(struct device *dev, const struct resource *res, return IOMEM_ERR_PTR(-EINVAL); } + if (type == DEVM_IOREMAP && res->flags & IORESOURCE_MEM_NONPOSTED) + type = DEVM_IOREMAP_NP; + size = resource_size(res); - if (!devm_request_mem_region(dev, res->start, size, dev_name(dev))) { + if (res->name) + pretty_name = devm_kasprintf(dev, GFP_KERNEL, "%s %s", + dev_name(dev), res->name); + else + pretty_name = devm_kstrdup(dev, dev_name(dev), GFP_KERNEL); + if (!pretty_name) { + dev_err(dev, "can't generate pretty name for resource %pR\n", res); + return IOMEM_ERR_PTR(-ENOMEM); + } + + if (!devm_request_mem_region(dev, res->start, size, pretty_name)) { dev_err(dev, "can't request region for resource %pR\n", res); return IOMEM_ERR_PTR(-EBUSY); } @@ -173,13 +186,15 @@ __devm_ioremap_resource(struct device *dev, const struct resource *res, * region and ioremaps it. All operations are managed and will be undone * on driver detach. * - * Returns a pointer to the remapped memory or an ERR_PTR() encoded error code - * on failure. Usage example: + * Usage example: * * res = platform_get_resource(pdev, IORESOURCE_MEM, 0); * base = devm_ioremap_resource(&pdev->dev, res); * if (IS_ERR(base)) * return PTR_ERR(base); + * + * Return: a pointer to the remapped memory or an ERR_PTR() encoded error code + * on failure. */ void __iomem *devm_ioremap_resource(struct device *dev, const struct resource *res) @@ -194,8 +209,8 @@ EXPORT_SYMBOL(devm_ioremap_resource); * @dev: generic device to handle the resource for * @res: resource to be handled * - * Returns a pointer to the remapped memory or an ERR_PTR() encoded error code - * on failure. Usage example: + * Return: a pointer to the remapped memory or an ERR_PTR() encoded error code + * on failure. */ void __iomem *devm_ioremap_resource_wc(struct device *dev, const struct resource *res) @@ -218,12 +233,20 @@ void __iomem *devm_ioremap_resource_wc(struct device *dev, * @node: The device-tree node where the resource resides * @index: index of the MMIO range in the "reg" property * @size: Returns the size of the resource (pass NULL if not needed) - * Returns a pointer to the requested and mapped memory or an ERR_PTR() encoded - * error code on failure. Usage example: + * + * Usage example: * * base = devm_of_iomap(&pdev->dev, node, 0, NULL); * if (IS_ERR(base)) * return PTR_ERR(base); + * + * Please Note: This is not a one-to-one replacement for of_iomap() because the + * of_iomap() function does not track whether the region is already mapped. If + * two drivers try to map the same memory, the of_iomap() function will succeed + * but the devm_of_iomap() function will return -EBUSY. + * + * Return: a pointer to the requested and mapped memory or an ERR_PTR() encoded + * error code on failure. */ void __iomem *devm_of_iomap(struct device *dev, struct device_node *node, int index, resource_size_t *size) @@ -261,6 +284,8 @@ static int devm_ioport_map_match(struct device *dev, void *res, * * Managed ioport_map(). Map is automatically unmapped on driver * detach. + * + * Return: a pointer to the remapped memory or NULL on failure. */ void __iomem *devm_ioport_map(struct device *dev, unsigned long port, unsigned int nr) @@ -330,7 +355,7 @@ static void pcim_iomap_release(struct device *gendev, void *res) * detach. * * This function might sleep when the table is first allocated but can - * be safely called without context and guaranteed to succed once + * be safely called without context and guaranteed to succeed once * allocated. */ void __iomem * const *pcim_iomap_table(struct pci_dev *pdev) @@ -503,3 +528,85 @@ void pcim_iounmap_regions(struct pci_dev *pdev, int mask) } EXPORT_SYMBOL(pcim_iounmap_regions); #endif /* CONFIG_PCI */ + +static void devm_arch_phys_ac_add_release(struct device *dev, void *res) +{ + arch_phys_wc_del(*((int *)res)); +} + +/** + * devm_arch_phys_wc_add - Managed arch_phys_wc_add() + * @dev: Managed device + * @base: Memory base address + * @size: Size of memory range + * + * Adds a WC MTRR using arch_phys_wc_add() and sets up a release callback. + * See arch_phys_wc_add() for more information. + */ +int devm_arch_phys_wc_add(struct device *dev, unsigned long base, unsigned long size) +{ + int *mtrr; + int ret; + + mtrr = devres_alloc(devm_arch_phys_ac_add_release, sizeof(*mtrr), GFP_KERNEL); + if (!mtrr) + return -ENOMEM; + + ret = arch_phys_wc_add(base, size); + if (ret < 0) { + devres_free(mtrr); + return ret; + } + + *mtrr = ret; + devres_add(dev, mtrr); + + return ret; +} +EXPORT_SYMBOL(devm_arch_phys_wc_add); + +struct arch_io_reserve_memtype_wc_devres { + resource_size_t start; + resource_size_t size; +}; + +static void devm_arch_io_free_memtype_wc_release(struct device *dev, void *res) +{ + const struct arch_io_reserve_memtype_wc_devres *this = res; + + arch_io_free_memtype_wc(this->start, this->size); +} + +/** + * devm_arch_io_reserve_memtype_wc - Managed arch_io_reserve_memtype_wc() + * @dev: Managed device + * @start: Memory base address + * @size: Size of memory range + * + * Reserves a memory range with WC caching using arch_io_reserve_memtype_wc() + * and sets up a release callback See arch_io_reserve_memtype_wc() for more + * information. + */ +int devm_arch_io_reserve_memtype_wc(struct device *dev, resource_size_t start, + resource_size_t size) +{ + struct arch_io_reserve_memtype_wc_devres *dr; + int ret; + + dr = devres_alloc(devm_arch_io_free_memtype_wc_release, sizeof(*dr), GFP_KERNEL); + if (!dr) + return -ENOMEM; + + ret = arch_io_reserve_memtype_wc(start, size); + if (ret < 0) { + devres_free(dr); + return ret; + } + + dr->start = start; + dr->size = size; + devres_add(dev, dr); + + return ret; +} +EXPORT_SYMBOL(devm_arch_io_reserve_memtype_wc); diff --git a/lib/digsig.c b/lib/digsig.c index e0627c3e53b2..04b5e55ed95f 100644 --- a/lib/digsig.c +++ b/lib/digsig.c @@ -20,7 +20,7 @@ #include <linux/key.h> #include <linux/crypto.h> #include <crypto/hash.h> -#include <crypto/sha.h> +#include <crypto/sha1.h> #include <keys/user-type.h> #include <linux/mpi.h> #include <linux/digsig.h> diff --git a/lib/dim/net_dim.c b/lib/dim/net_dim.c index a4db51c21266..06811d866775 100644 --- a/lib/dim/net_dim.c +++ b/lib/dim/net_dim.c @@ -233,7 +233,7 @@ void net_dim(struct dim *dim, struct dim_sample end_sample) schedule_work(&dim->work); break; } - /* fall through */ + fallthrough; case DIM_START_MEASURE: dim_update_sample(end_sample.event_ctr, end_sample.pkt_ctr, end_sample.byte_ctr, &dim->start_sample); diff --git a/lib/dim/rdma_dim.c b/lib/dim/rdma_dim.c index f7e26c7b4749..15462d54758d 100644 --- a/lib/dim/rdma_dim.c +++ b/lib/dim/rdma_dim.c @@ -59,7 +59,7 @@ static bool rdma_dim_decision(struct dim_stats *curr_stats, struct dim *dim) break; case DIM_STATS_WORSE: dim_turn(dim); - /* fall through */ + fallthrough; case DIM_STATS_BETTER: step_res = rdma_dim_step(dim); if (step_res == DIM_ON_EDGE) @@ -94,7 +94,7 @@ void rdma_dim(struct dim *dim, u64 completions) schedule_work(&dim->work); break; } - /* fall through */ + fallthrough; case DIM_START_MEASURE: dim->state = DIM_MEASURE_IN_PROGRESS; dim_update_sample_with_comps(curr_sample->event_ctr, 0, 0, diff --git a/lib/dump_stack.c b/lib/dump_stack.c index 33ffbf308853..6b7f1bf6715d 100644 --- a/lib/dump_stack.c +++ b/lib/dump_stack.c @@ -5,6 +5,7 @@ */ #include <linux/kernel.h> +#include <linux/buildid.h> #include <linux/export.h> #include <linux/sched.h> #include <linux/sched/debug.h> @@ -12,6 +13,7 @@ #include <linux/atomic.h> #include <linux/kexec.h> #include <linux/utsname.h> +#include <linux/stop_machine.h> static char dump_stack_arch_desc_str[128]; @@ -35,6 +37,14 @@ void __init dump_stack_set_arch_desc(const char *fmt, ...) va_end(args); } +#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) +#define BUILD_ID_FMT " %20phN" +#define BUILD_ID_VAL vmlinux_build_id +#else +#define BUILD_ID_FMT "%s" +#define BUILD_ID_VAL "" +#endif + /** * dump_stack_print_info - print generic debug info for dump_stack() * @log_lvl: log level @@ -44,19 +54,20 @@ void __init dump_stack_set_arch_desc(const char *fmt, ...) */ void dump_stack_print_info(const char *log_lvl) { - printk("%sCPU: %d PID: %d Comm: %.20s %s%s %s %.*s\n", + printk("%sCPU: %d PID: %d Comm: %.20s %s%s %s %.*s" BUILD_ID_FMT "\n", log_lvl, raw_smp_processor_id(), current->pid, current->comm, kexec_crash_loaded() ? "Kdump: loaded " : "", print_tainted(), init_utsname()->release, (int)strcspn(init_utsname()->version, " "), - init_utsname()->version); + init_utsname()->version, BUILD_ID_VAL); if (dump_stack_arch_desc_str[0] != '\0') printk("%sHardware name: %s\n", log_lvl, dump_stack_arch_desc_str); print_worker_info(log_lvl, current); + print_stop_info(log_lvl, current); } /** @@ -71,61 +82,34 @@ void show_regs_print_info(const char *log_lvl) dump_stack_print_info(log_lvl); } -static void __dump_stack(void) +static void __dump_stack(const char *log_lvl) { - dump_stack_print_info(KERN_DEFAULT); - show_stack(NULL, NULL); + dump_stack_print_info(log_lvl); + show_stack(NULL, NULL, log_lvl); } /** - * dump_stack - dump the current task information and its stack trace + * dump_stack_lvl - dump the current task information and its stack trace + * @log_lvl: log level * * Architectures can override this implementation by implementing its own. */ -#ifdef CONFIG_SMP -static atomic_t dump_lock = ATOMIC_INIT(-1); - -asmlinkage __visible void dump_stack(void) +asmlinkage __visible void dump_stack_lvl(const char *log_lvl) { unsigned long flags; - int was_locked; - int old; - int cpu; /* * Permit this cpu to perform nested stack dumps while serialising * against other CPUs */ -retry: - local_irq_save(flags); - cpu = smp_processor_id(); - old = atomic_cmpxchg(&dump_lock, -1, cpu); - if (old == -1) { - was_locked = 0; - } else if (old == cpu) { - was_locked = 1; - } else { - local_irq_restore(flags); - /* - * Wait for the lock to release before jumping to - * atomic_cmpxchg() in order to mitigate the thundering herd - * problem. - */ - do { cpu_relax(); } while (atomic_read(&dump_lock) != -1); - goto retry; - } - - __dump_stack(); - - if (!was_locked) - atomic_set(&dump_lock, -1); - - local_irq_restore(flags); + printk_cpu_lock_irqsave(flags); + __dump_stack(log_lvl); + printk_cpu_unlock_irqrestore(flags); } -#else +EXPORT_SYMBOL(dump_stack_lvl); + asmlinkage __visible void dump_stack(void) { - __dump_stack(); + dump_stack_lvl(KERN_DEFAULT); } -#endif EXPORT_SYMBOL(dump_stack); diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index c60409138e13..dd7f56af9aed 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -11,7 +11,7 @@ * Copyright (C) 2013 Du, Changbin <changbin.du@gmail.com> */ -#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ +#define pr_fmt(fmt) "dyndbg: " fmt #include <linux/kernel.h> #include <linux/module.h> @@ -39,8 +39,8 @@ #include <rdma/ib_verbs.h> -extern struct _ddebug __start___verbose[]; -extern struct _ddebug __stop___verbose[]; +extern struct _ddebug __start___dyndbg[]; +extern struct _ddebug __stop___dyndbg[]; struct ddebug_table { struct list_head link; @@ -62,10 +62,17 @@ struct ddebug_iter { unsigned int idx; }; +struct flag_settings { + unsigned int flags; + unsigned int mask; +}; + static DEFINE_MUTEX(ddebug_lock); static LIST_HEAD(ddebug_tables); static int verbose; module_param(verbose, int, 0644); +MODULE_PARM_DESC(verbose, " dynamic_debug/control processing " + "( 0 = off (default), 1 = module add/rm, 2 = >control summary, 3 = parsing, 4 = per-site changes)"); /* Return the path relative to source root */ static inline const char *trim_prefix(const char *path) @@ -87,30 +94,35 @@ static struct { unsigned flag:8; char opt_char; } opt_array[] = { { _DPRINTK_FLAGS_NONE, '_' }, }; +struct flagsbuf { char buf[ARRAY_SIZE(opt_array)+1]; }; + /* format a string into buf[] which describes the _ddebug's flags */ -static char *ddebug_describe_flags(struct _ddebug *dp, char *buf, - size_t maxlen) +static char *ddebug_describe_flags(unsigned int flags, struct flagsbuf *fb) { - char *p = buf; + char *p = fb->buf; int i; - BUG_ON(maxlen < 6); for (i = 0; i < ARRAY_SIZE(opt_array); ++i) - if (dp->flags & opt_array[i].flag) + if (flags & opt_array[i].flag) *p++ = opt_array[i].opt_char; - if (p == buf) + if (p == fb->buf) *p++ = '_'; *p = '\0'; - return buf; + return fb->buf; } -#define vpr_info(fmt, ...) \ +#define vnpr_info(lvl, fmt, ...) \ do { \ - if (verbose) \ + if (verbose >= lvl) \ pr_info(fmt, ##__VA_ARGS__); \ } while (0) +#define vpr_info(fmt, ...) vnpr_info(1, fmt, ##__VA_ARGS__) +#define v2pr_info(fmt, ...) vnpr_info(2, fmt, ##__VA_ARGS__) +#define v3pr_info(fmt, ...) vnpr_info(3, fmt, ##__VA_ARGS__) +#define v4pr_info(fmt, ...) vnpr_info(4, fmt, ##__VA_ARGS__) + static void vpr_info_dq(const struct ddebug_query *query, const char *msg) { /* trim any trailing newlines */ @@ -122,12 +134,12 @@ static void vpr_info_dq(const struct ddebug_query *query, const char *msg) fmtlen--; } - vpr_info("%s: func=\"%s\" file=\"%s\" module=\"%s\" format=\"%.*s\" lineno=%u-%u\n", + v3pr_info("%s: func=\"%s\" file=\"%s\" module=\"%s\" format=\"%.*s\" lineno=%u-%u\n", msg, - query->function ? query->function : "", - query->filename ? query->filename : "", - query->module ? query->module : "", - fmtlen, query->format ? query->format : "", + query->function ?: "", + query->filename ?: "", + query->module ?: "", + fmtlen, query->format ?: "", query->first_lineno, query->last_lineno); } @@ -138,13 +150,13 @@ static void vpr_info_dq(const struct ddebug_query *query, const char *msg) * logs the changes. Takes ddebug_lock. */ static int ddebug_change(const struct ddebug_query *query, - unsigned int flags, unsigned int mask) + struct flag_settings *modifiers) { int i; struct ddebug_table *dt; unsigned int newflags; unsigned int nfound = 0; - char flagbuf[10]; + struct flagsbuf fbuf; /* search for matching ddebugs */ mutex_lock(&ddebug_lock); @@ -173,9 +185,16 @@ static int ddebug_change(const struct ddebug_query *query, continue; /* match against the format */ - if (query->format && - !strstr(dp->format, query->format)) - continue; + if (query->format) { + if (*query->format == '^') { + char *p; + /* anchored search. match must be at beginning */ + p = strstr(dp->format, query->format+1); + if (p != dp->format) + continue; + } else if (!strstr(dp->format, query->format)) + continue; + } /* match against the line number range */ if (query->first_lineno && @@ -187,22 +206,21 @@ static int ddebug_change(const struct ddebug_query *query, nfound++; - newflags = (dp->flags & mask) | flags; + newflags = (dp->flags & modifiers->mask) | modifiers->flags; if (newflags == dp->flags) continue; #ifdef CONFIG_JUMP_LABEL if (dp->flags & _DPRINTK_FLAGS_PRINT) { - if (!(flags & _DPRINTK_FLAGS_PRINT)) + if (!(modifiers->flags & _DPRINTK_FLAGS_PRINT)) static_branch_disable(&dp->key.dd_key_true); - } else if (flags & _DPRINTK_FLAGS_PRINT) + } else if (modifiers->flags & _DPRINTK_FLAGS_PRINT) static_branch_enable(&dp->key.dd_key_true); #endif dp->flags = newflags; - vpr_info("changed %s:%d [%s]%s =%s\n", + v4pr_info("changed %s:%d [%s]%s =%s\n", trim_prefix(dp->filename), dp->lineno, dt->mod_name, dp->function, - ddebug_describe_flags(dp, flagbuf, - sizeof(flagbuf))); + ddebug_describe_flags(dp->flags, &fbuf)); } } mutex_unlock(&ddebug_lock); @@ -259,7 +277,7 @@ static int ddebug_tokenize(char *buf, char *words[], int maxwords) buf = end; } - if (verbose) { + if (verbose >= 3) { int i; pr_info("split into words:"); for (i = 0; i < nwords; i++) @@ -289,6 +307,41 @@ static inline int parse_lineno(const char *str, unsigned int *val) return 0; } +static int parse_linerange(struct ddebug_query *query, const char *first) +{ + char *last = strchr(first, '-'); + + if (query->first_lineno || query->last_lineno) { + pr_err("match-spec: line used 2x\n"); + return -EINVAL; + } + if (last) + *last++ = '\0'; + if (parse_lineno(first, &query->first_lineno) < 0) + return -EINVAL; + if (last) { + /* range <first>-<last> */ + if (parse_lineno(last, &query->last_lineno) < 0) + return -EINVAL; + + /* special case for last lineno not specified */ + if (query->last_lineno == 0) + query->last_lineno = UINT_MAX; + + if (query->last_lineno < query->first_lineno) { + pr_err("last-line:%d < 1st-line:%d\n", + query->last_lineno, + query->first_lineno); + return -EINVAL; + } + } else { + query->last_lineno = query->first_lineno; + } + v3pr_info("parsed line %d-%d\n", query->first_lineno, + query->last_lineno); + return 0; +} + static int check_set(const char **dest, char *src, char *name) { int rc = 0; @@ -322,61 +375,53 @@ static int ddebug_parse_query(char *words[], int nwords, { unsigned int i; int rc = 0; + char *fline; /* check we have an even number of words */ if (nwords % 2 != 0) { pr_err("expecting pairs of match-spec <value>\n"); return -EINVAL; } - memset(query, 0, sizeof(*query)); if (modname) /* support $modname.dyndbg=<multiple queries> */ query->module = modname; for (i = 0; i < nwords; i += 2) { - if (!strcmp(words[i], "func")) { - rc = check_set(&query->function, words[i+1], "func"); - } else if (!strcmp(words[i], "file")) { - rc = check_set(&query->filename, words[i+1], "file"); - } else if (!strcmp(words[i], "module")) { - rc = check_set(&query->module, words[i+1], "module"); - } else if (!strcmp(words[i], "format")) { - string_unescape_inplace(words[i+1], UNESCAPE_SPACE | - UNESCAPE_OCTAL | - UNESCAPE_SPECIAL); - rc = check_set(&query->format, words[i+1], "format"); - } else if (!strcmp(words[i], "line")) { - char *first = words[i+1]; - char *last = strchr(first, '-'); - if (query->first_lineno || query->last_lineno) { - pr_err("match-spec: line used 2x\n"); - return -EINVAL; - } - if (last) - *last++ = '\0'; - if (parse_lineno(first, &query->first_lineno) < 0) - return -EINVAL; - if (last) { - /* range <first>-<last> */ - if (parse_lineno(last, &query->last_lineno) < 0) - return -EINVAL; + char *keyword = words[i]; + char *arg = words[i+1]; - /* special case for last lineno not specified */ - if (query->last_lineno == 0) - query->last_lineno = UINT_MAX; + if (!strcmp(keyword, "func")) { + rc = check_set(&query->function, arg, "func"); + } else if (!strcmp(keyword, "file")) { + if (check_set(&query->filename, arg, "file")) + return -EINVAL; - if (query->last_lineno < query->first_lineno) { - pr_err("last-line:%d < 1st-line:%d\n", - query->last_lineno, - query->first_lineno); + /* tail :$info is function or line-range */ + fline = strchr(query->filename, ':'); + if (!fline) + continue; + *fline++ = '\0'; + if (isalpha(*fline) || *fline == '*' || *fline == '?') { + /* take as function name */ + if (check_set(&query->function, fline, "func")) return -EINVAL; - } } else { - query->last_lineno = query->first_lineno; + if (parse_linerange(query, fline)) + return -EINVAL; } + } else if (!strcmp(keyword, "module")) { + rc = check_set(&query->module, arg, "module"); + } else if (!strcmp(keyword, "format")) { + string_unescape_inplace(arg, UNESCAPE_SPACE | + UNESCAPE_OCTAL | + UNESCAPE_SPECIAL); + rc = check_set(&query->format, arg, "format"); + } else if (!strcmp(keyword, "line")) { + if (parse_linerange(query, arg)) + return -EINVAL; } else { - pr_err("unknown keyword \"%s\"\n", words[i]); + pr_err("unknown keyword \"%s\"\n", keyword); return -EINVAL; } if (rc) @@ -392,11 +437,9 @@ static int ddebug_parse_query(char *words[], int nwords, * flags fields of matched _ddebug's. Returns 0 on success * or <0 on error. */ -static int ddebug_parse_flags(const char *str, unsigned int *flagsp, - unsigned int *maskp) +static int ddebug_parse_flags(const char *str, struct flag_settings *modifiers) { - unsigned flags = 0; - int op = '=', i; + int op, i; switch (*str) { case '+': @@ -408,45 +451,45 @@ static int ddebug_parse_flags(const char *str, unsigned int *flagsp, pr_err("bad flag-op %c, at start of %s\n", *str, str); return -EINVAL; } - vpr_info("op='%c'\n", op); + v3pr_info("op='%c'\n", op); for (; *str ; ++str) { for (i = ARRAY_SIZE(opt_array) - 1; i >= 0; i--) { if (*str == opt_array[i].opt_char) { - flags |= opt_array[i].flag; + modifiers->flags |= opt_array[i].flag; break; } } if (i < 0) { - pr_err("unknown flag '%c' in \"%s\"\n", *str, str); + pr_err("unknown flag '%c'\n", *str); return -EINVAL; } } - vpr_info("flags=0x%x\n", flags); + v3pr_info("flags=0x%x\n", modifiers->flags); - /* calculate final *flagsp, *maskp according to mask and op */ + /* calculate final flags, mask based upon op */ switch (op) { case '=': - *maskp = 0; - *flagsp = flags; + /* modifiers->flags already set */ + modifiers->mask = 0; break; case '+': - *maskp = ~0U; - *flagsp = flags; + modifiers->mask = ~0U; break; case '-': - *maskp = ~flags; - *flagsp = 0; + modifiers->mask = ~modifiers->flags; + modifiers->flags = 0; break; } - vpr_info("*flagsp=0x%x *maskp=0x%x\n", *flagsp, *maskp); + v3pr_info("*flagsp=0x%x *maskp=0x%x\n", modifiers->flags, modifiers->mask); + return 0; } static int ddebug_exec_query(char *query_string, const char *modname) { - unsigned int flags = 0, mask = 0; - struct ddebug_query query; + struct flag_settings modifiers = {}; + struct ddebug_query query = {}; #define MAXWORDS 9 int nwords, nfound; char *words[MAXWORDS]; @@ -457,7 +500,7 @@ static int ddebug_exec_query(char *query_string, const char *modname) return -EINVAL; } /* check flags 1st (last arg) so query is pairs of spec,val */ - if (ddebug_parse_flags(words[nwords-1], &flags, &mask)) { + if (ddebug_parse_flags(words[nwords-1], &modifiers)) { pr_err("flags parse failed\n"); return -EINVAL; } @@ -466,7 +509,7 @@ static int ddebug_exec_query(char *query_string, const char *modname) return -EINVAL; } /* actually go and implement the change */ - nfound = ddebug_change(&query, flags, mask); + nfound = ddebug_change(&query, &modifiers); vpr_info_dq(&query, nfound ? "applied" : "no-match"); return nfound; @@ -490,7 +533,7 @@ static int ddebug_exec_queries(char *query, const char *modname) if (!query || !*query || *query == '#') continue; - vpr_info("query %d: \"%s\"\n", i, query); + vpr_info("query %d: \"%s\" mod:%s\n", i, query, modname ?: "*"); rc = ddebug_exec_query(query, modname); if (rc < 0) { @@ -501,14 +544,44 @@ static int ddebug_exec_queries(char *query, const char *modname) } i++; } - vpr_info("processed %d queries, with %d matches, %d errs\n", - i, nfound, errs); + if (i) + v2pr_info("processed %d queries, with %d matches, %d errs\n", + i, nfound, errs); if (exitcode) return exitcode; return nfound; } +/** + * dynamic_debug_exec_queries - select and change dynamic-debug prints + * @query: query-string described in admin-guide/dynamic-debug-howto + * @modname: string containing module name, usually &module.mod_name + * + * This uses the >/proc/dynamic_debug/control reader, allowing module + * authors to modify their dynamic-debug callsites. The modname is + * canonically struct module.mod_name, but can also be null or a + * module-wildcard, for example: "drm*". + */ +int dynamic_debug_exec_queries(const char *query, const char *modname) +{ + int rc; + char *qry; /* writable copy of query */ + + if (!query) { + pr_err("non-null query/command string expected\n"); + return -EINVAL; + } + qry = kstrndup(query, PAGE_SIZE, GFP_KERNEL); + if (!qry) + return -ENOMEM; + + rc = ddebug_exec_queries(qry, modname); + kfree(qry); + return rc; +} +EXPORT_SYMBOL_GPL(dynamic_debug_exec_queries); + #define PREFIX_SIZE 64 static int remaining(int wrote) @@ -518,13 +591,11 @@ static int remaining(int wrote) return 0; } -static char *dynamic_emit_prefix(const struct _ddebug *desc, char *buf) +static char *__dynamic_emit_prefix(const struct _ddebug *desc, char *buf) { int pos_after_tid; int pos = 0; - *buf = '\0'; - if (desc->flags & _DPRINTK_FLAGS_INCL_TID) { if (in_interrupt()) pos += snprintf(buf + pos, remaining(pos), "<intr> "); @@ -550,11 +621,18 @@ static char *dynamic_emit_prefix(const struct _ddebug *desc, char *buf) return buf; } +static inline char *dynamic_emit_prefix(struct _ddebug *desc, char *buf) +{ + if (unlikely(desc->flags & _DPRINTK_FLAGS_INCL_ANY)) + return __dynamic_emit_prefix(desc, buf); + return buf; +} + void __dynamic_pr_debug(struct _ddebug *descriptor, const char *fmt, ...) { va_list args; struct va_format vaf; - char buf[PREFIX_SIZE]; + char buf[PREFIX_SIZE] = ""; BUG_ON(!descriptor); BUG_ON(!fmt); @@ -587,7 +665,7 @@ void __dynamic_dev_dbg(struct _ddebug *descriptor, if (!dev) { printk(KERN_DEBUG "(NULL device *): %pV", &vaf); } else { - char buf[PREFIX_SIZE]; + char buf[PREFIX_SIZE] = ""; dev_printk_emit(LOGLEVEL_DEBUG, dev, "%s%s %s: %pV", dynamic_emit_prefix(descriptor, buf), @@ -616,7 +694,7 @@ void __dynamic_netdev_dbg(struct _ddebug *descriptor, vaf.va = &args; if (dev && dev->dev.parent) { - char buf[PREFIX_SIZE]; + char buf[PREFIX_SIZE] = ""; dev_printk_emit(LOGLEVEL_DEBUG, dev->dev.parent, "%s%s %s %s%s: %pV", @@ -652,7 +730,7 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, vaf.va = &args; if (ibdev && ibdev->dev.parent) { - char buf[PREFIX_SIZE]; + char buf[PREFIX_SIZE] = ""; dev_printk_emit(LOGLEVEL_DEBUG, ibdev->dev.parent, "%s%s %s %s: %pV", @@ -673,20 +751,17 @@ EXPORT_SYMBOL(__dynamic_ibdev_dbg); #endif -#define DDEBUG_STRING_SIZE 1024 -static __initdata char ddebug_setup_string[DDEBUG_STRING_SIZE]; - -static __init int ddebug_setup_query(char *str) +/* + * Install a noop handler to make dyndbg look like a normal kernel cli param. + * This avoids warnings about dyndbg being an unknown cli param when supplied + * by a user. + */ +static __init int dyndbg_setup(char *str) { - if (strlen(str) >= DDEBUG_STRING_SIZE) { - pr_warn("ddebug boot param string too large\n"); - return 0; - } - strlcpy(ddebug_setup_string, str, DDEBUG_STRING_SIZE); return 1; } -__setup("ddebug_query=", ddebug_setup_query); +__setup("dyndbg=", dyndbg_setup); /* * File_ops->write method for <debugfs>/dynamic_debug/control. Gathers the @@ -708,7 +783,7 @@ static ssize_t ddebug_proc_write(struct file *file, const char __user *ubuf, tmpbuf = memdup_user_nul(ubuf, len); if (IS_ERR(tmpbuf)) return PTR_ERR(tmpbuf); - vpr_info("read %d bytes from userspace\n", (int)len); + v2pr_info("read %zu bytes from userspace\n", len); ret = ddebug_exec_queries(tmpbuf, NULL); kfree(tmpbuf); @@ -771,8 +846,6 @@ static void *ddebug_proc_start(struct seq_file *m, loff_t *pos) struct _ddebug *dp; int n = *pos; - vpr_info("called m=%p *pos=%lld\n", m, (unsigned long long)*pos); - mutex_lock(&ddebug_lock); if (!n) @@ -795,9 +868,6 @@ static void *ddebug_proc_next(struct seq_file *m, void *p, loff_t *pos) struct ddebug_iter *iter = m->private; struct _ddebug *dp; - vpr_info("called m=%p p=%p *pos=%lld\n", - m, p, (unsigned long long)*pos); - if (p == SEQ_START_TOKEN) dp = ddebug_iter_first(iter); else @@ -816,9 +886,7 @@ static int ddebug_proc_show(struct seq_file *m, void *p) { struct ddebug_iter *iter = m->private; struct _ddebug *dp = p; - char flagsbuf[10]; - - vpr_info("called m=%p p=%p\n", m, p); + struct flagsbuf flags; if (p == SEQ_START_TOKEN) { seq_puts(m, @@ -829,7 +897,7 @@ static int ddebug_proc_show(struct seq_file *m, void *p) seq_printf(m, "%s:%u [%s]%s =%s \"", trim_prefix(dp->filename), dp->lineno, iter->table->mod_name, dp->function, - ddebug_describe_flags(dp, flagsbuf, sizeof(flagsbuf))); + ddebug_describe_flags(dp->flags, &flags)); seq_escape(m, dp->format, "\t\r\n\""); seq_puts(m, "\"\n"); @@ -842,7 +910,6 @@ static int ddebug_proc_show(struct seq_file *m, void *p) */ static void ddebug_proc_stop(struct seq_file *m, void *p) { - vpr_info("called m=%p p=%p\n", m, p); mutex_unlock(&ddebug_lock); } @@ -853,16 +920,8 @@ static const struct seq_operations ddebug_proc_seqops = { .stop = ddebug_proc_stop }; -/* - * File_ops->open method for <debugfs>/dynamic_debug/control. Does - * the seq_file setup dance, and also creates an iterator to walk the - * _ddebugs. Note that we create a seq_file always, even for O_WRONLY - * files where it's not needed, as doing so simplifies the ->release - * method. - */ static int ddebug_proc_open(struct inode *inode, struct file *file) { - vpr_info("called\n"); return seq_open_private(file, &ddebug_proc_seqops, sizeof(struct ddebug_iter)); } @@ -876,6 +935,14 @@ static const struct file_operations ddebug_proc_fops = { .write = ddebug_proc_write }; +static const struct proc_ops proc_fops = { + .proc_open = ddebug_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = seq_release_private, + .proc_write = ddebug_proc_write +}; + /* * Allocate a new ddebug_table for the given module * and add it to the global list. @@ -901,10 +968,10 @@ int ddebug_add_module(struct _ddebug *tab, unsigned int n, dt->ddebugs = tab; mutex_lock(&ddebug_lock); - list_add_tail(&dt->link, &ddebug_tables); + list_add(&dt->link, &ddebug_tables); mutex_unlock(&ddebug_lock); - vpr_info("%u debug prints in module %s\n", n, dt->mod_name); + vpr_info("%3u debug prints in module %s\n", n, dt->mod_name); return 0; } @@ -926,7 +993,7 @@ static int ddebug_dyndbg_param_cb(char *param, char *val, ddebug_exec_queries((val ? val : "+p"), modname); - return 0; /* query failure shouldnt stop module load */ + return 0; /* query failure shouldn't stop module load */ } /* handle both dyndbg and $module.dyndbg params at boot */ @@ -963,8 +1030,6 @@ int ddebug_remove_module(const char *mod_name) struct ddebug_table *dt, *nextdt; int ret = -ENOENT; - vpr_info("removing module \"%s\"\n", mod_name); - mutex_lock(&ddebug_lock); list_for_each_entry_safe(dt, nextdt, &ddebug_tables, link) { if (dt->mod_name == mod_name) { @@ -974,6 +1039,8 @@ int ddebug_remove_module(const char *mod_name) } } mutex_unlock(&ddebug_lock); + if (!ret) + v2pr_info("removed module \"%s\"\n", mod_name); return ret; } @@ -991,15 +1058,25 @@ static void ddebug_remove_all_tables(void) static __initdata int ddebug_init_success; -static int __init dynamic_debug_init_debugfs(void) +static int __init dynamic_debug_init_control(void) { - struct dentry *dir; + struct proc_dir_entry *procfs_dir; + struct dentry *debugfs_dir; if (!ddebug_init_success) return -ENODEV; - dir = debugfs_create_dir("dynamic_debug", NULL); - debugfs_create_file("control", 0644, dir, NULL, &ddebug_proc_fops); + /* Create the control file in debugfs if it is enabled */ + if (debugfs_initialized()) { + debugfs_dir = debugfs_create_dir("dynamic_debug", NULL); + debugfs_create_file("control", 0644, debugfs_dir, NULL, + &ddebug_proc_fops); + } + + /* Also create the control file in procfs */ + procfs_dir = proc_mkdir("dynamic_debug", NULL); + if (procfs_dir) + proc_create("control", 0644, procfs_dir, &proc_fops); return 0; } @@ -1011,20 +1088,21 @@ static int __init dynamic_debug_init(void) char *cmdline; int ret = 0; int n = 0, entries = 0, modct = 0; - int verbose_bytes = 0; - if (__start___verbose == __stop___verbose) { - pr_warn("_ddebug table is empty in a CONFIG_DYNAMIC_DEBUG build\n"); - return 1; + if (&__start___dyndbg == &__stop___dyndbg) { + if (IS_ENABLED(CONFIG_DYNAMIC_DEBUG)) { + pr_warn("_ddebug table is empty in a CONFIG_DYNAMIC_DEBUG build\n"); + return 1; + } + pr_info("Ignore empty _ddebug table in a CONFIG_DYNAMIC_DEBUG_CORE build\n"); + ddebug_init_success = 1; + return 0; } - iter = __start___verbose; + iter = __start___dyndbg; modname = iter->modname; iter_start = iter; - for (; iter < __stop___verbose; iter++) { + for (; iter < __stop___dyndbg; iter++) { entries++; - verbose_bytes += strlen(iter->modname) + strlen(iter->function) - + strlen(iter->filename) + strlen(iter->format); - if (strcmp(modname, iter->modname)) { modct++; ret = ddebug_add_module(iter_start, n, modname); @@ -1041,20 +1119,10 @@ static int __init dynamic_debug_init(void) goto out_err; ddebug_init_success = 1; - vpr_info("%d modules, %d entries and %d bytes in ddebug tables, %d bytes in (readonly) verbose section\n", - modct, entries, (int)(modct * sizeof(struct ddebug_table)), - verbose_bytes + (int)(__stop___verbose - __start___verbose)); - - /* apply ddebug_query boot param, dont unload tables on err */ - if (ddebug_setup_string[0] != '\0') { - pr_warn("ddebug_query param name is deprecated, change it to dyndbg\n"); - ret = ddebug_exec_queries(ddebug_setup_string, NULL); - if (ret < 0) - pr_warn("Invalid ddebug boot param %s\n", - ddebug_setup_string); - else - pr_info("%d changes by ddebug_query\n", ret); - } + vpr_info("%d prdebugs in %d modules, %d KiB in ddebug tables, %d kiB in __dyndbg section\n", + entries, modct, (int)((modct * sizeof(struct ddebug_table)) >> 10), + (int)((entries * sizeof(struct _ddebug)) >> 10)); + /* now that ddebug tables are loaded, process all boot args * again to find and activate queries given in dyndbg params. * While this has already been done for known boot params, it @@ -1077,4 +1145,4 @@ out_err: early_initcall(dynamic_debug_init); /* Debugfs setup must be done later */ -fs_initcall(dynamic_debug_init_debugfs); +fs_initcall(dynamic_debug_init_control); diff --git a/lib/dynamic_queue_limits.c b/lib/dynamic_queue_limits.c index e659a027036e..fde0aa244148 100644 --- a/lib/dynamic_queue_limits.c +++ b/lib/dynamic_queue_limits.c @@ -60,8 +60,8 @@ void dql_completed(struct dql *dql, unsigned int count) * A decrease is only considered if the queue has been busy in * the whole interval (the check above). * - * If there is slack, the amount of execess data queued above - * the the amount needed to prevent starvation, the queue limit + * If there is slack, the amount of excess data queued above + * the amount needed to prevent starvation, the queue limit * can be decreased. To avoid hysteresis we consider the * minimum amount of slack found over several iterations of the * completion routine. diff --git a/lib/earlycpio.c b/lib/earlycpio.c index c001e084829e..7921193f0424 100644 --- a/lib/earlycpio.c +++ b/lib/earlycpio.c @@ -40,16 +40,16 @@ enum cpio_fields { }; /** - * cpio_data find_cpio_data - Search for files in an uncompressed cpio + * find_cpio_data - Search for files in an uncompressed cpio * @path: The directory to search for, including a slash at the end - * @data: Pointer to the the cpio archive or a header inside + * @data: Pointer to the cpio archive or a header inside * @len: Remaining length of the cpio based on data pointer * @nextoff: When a matching file is found, this is the offset from the * beginning of the cpio to the beginning of the next file, not the * matching file itself. It can be used to iterate through the cpio * to find all files inside of a directory path. * - * @return: struct cpio_data containing the address, length and + * Return: &struct cpio_data containing the address, length and * filename (with the directory path cut off) of the found file. * If you search for a filename and not for files in a directory, * pass the absolute path of the filename in the cpio and make sure diff --git a/lib/errname.c b/lib/errname.c index 0c4d3e66170e..05cbf731545f 100644 --- a/lib/errname.c +++ b/lib/errname.c @@ -3,6 +3,7 @@ #include <linux/errno.h> #include <linux/errname.h> #include <linux/kernel.h> +#include <linux/math.h> /* * Ensure these tables do not accidentally become gigantic if some diff --git a/lib/error-inject.c b/lib/error-inject.c index aa63751c916f..2ff5ef689d72 100644 --- a/lib/error-inject.c +++ b/lib/error-inject.c @@ -8,6 +8,7 @@ #include <linux/mutex.h> #include <linux/list.h> #include <linux/slab.h> +#include <asm/sections.h> /* Whitelist of symbols that can be overridden for error injection. */ static LIST_HEAD(error_injection_list); @@ -64,7 +65,7 @@ static void populate_error_injection_list(struct error_injection_entry *start, mutex_lock(&ei_mutex); for (iter = start; iter < end; iter++) { - entry = arch_deref_entry_point((void *)iter->addr); + entry = (unsigned long)dereference_symbol_descriptor((void *)iter->addr); if (!kernel_text_address(entry) || !kallsyms_lookup_size_offset(entry, &size, &offset)) { @@ -180,6 +181,8 @@ static const char *error_type_string(int etype) return "ERRNO"; case EI_ETYPE_ERRNO_NULL: return "ERRNO_NULL"; + case EI_ETYPE_TRUE: + return "TRUE"; default: return "(unknown)"; } diff --git a/lib/errseq.c b/lib/errseq.c index 81f9e33aa7e7..93e9b94358dc 100644 --- a/lib/errseq.c +++ b/lib/errseq.c @@ -3,6 +3,7 @@ #include <linux/bug.h> #include <linux/atomic.h> #include <linux/errseq.h> +#include <linux/log2.h> /* * An errseq_t is a way of recording errors in one place, and allowing any diff --git a/lib/extable.c b/lib/extable.c index c3e59caf7ffa..9c9f40bd2b3d 100644 --- a/lib/extable.c +++ b/lib/extable.c @@ -21,7 +21,6 @@ static inline unsigned long ex_to_insn(const struct exception_table_entry *x) } #endif -#ifndef ARCH_HAS_SORT_EXTABLE #ifndef ARCH_HAS_RELATIVE_EXTABLE #define swap_ex NULL #else @@ -88,9 +87,6 @@ void trim_init_extable(struct module *m) m->num_exentries--; } #endif /* CONFIG_MODULES */ -#endif /* !ARCH_HAS_SORT_EXTABLE */ - -#ifndef ARCH_HAS_SEARCH_EXTABLE static int cmp_ex_search(const void *key, const void *elt) { @@ -120,4 +116,3 @@ search_extable(const struct exception_table_entry *base, return bsearch(&value, base, num, sizeof(struct exception_table_entry), cmp_ex_search); } -#endif diff --git a/lib/fault-inject-usercopy.c b/lib/fault-inject-usercopy.c new file mode 100644 index 000000000000..77558b6c29ca --- /dev/null +++ b/lib/fault-inject-usercopy.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/fault-inject.h> +#include <linux/fault-inject-usercopy.h> + +static struct { + struct fault_attr attr; +} fail_usercopy = { + .attr = FAULT_ATTR_INITIALIZER, +}; + +static int __init setup_fail_usercopy(char *str) +{ + return setup_fault_attr(&fail_usercopy.attr, str); +} +__setup("fail_usercopy=", setup_fail_usercopy); + +#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS + +static int __init fail_usercopy_debugfs(void) +{ + struct dentry *dir; + + dir = fault_create_debugfs_attr("fail_usercopy", NULL, + &fail_usercopy.attr); + if (IS_ERR(dir)) + return PTR_ERR(dir); + + return 0; +} + +late_initcall(fail_usercopy_debugfs); + +#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ + +bool should_fail_usercopy(void) +{ + return should_fail(&fail_usercopy.attr, 1); +} +EXPORT_SYMBOL_GPL(should_fail_usercopy); diff --git a/lib/fault-inject.c b/lib/fault-inject.c index 8186ca84910b..ce12621b4275 100644 --- a/lib/fault-inject.c +++ b/lib/fault-inject.c @@ -106,7 +106,9 @@ bool should_fail(struct fault_attr *attr, ssize_t size) unsigned int fail_nth = READ_ONCE(current->fail_nth); if (fail_nth) { - if (!WRITE_ONCE(current->fail_nth, fail_nth - 1)) + fail_nth--; + WRITE_ONCE(current->fail_nth, fail_nth); + if (!fail_nth) goto fail; return false; diff --git a/lib/fdt_addresses.c b/lib/fdt_addresses.c new file mode 100644 index 000000000000..23610bcf390b --- /dev/null +++ b/lib/fdt_addresses.c @@ -0,0 +1,2 @@ +#include <linux/libfdt_env.h> +#include "../scripts/dtc/libfdt/fdt_addresses.c" diff --git a/lib/find_bit.c b/lib/find_bit.c index e35a76b291e6..0f8e2e369b1d 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -15,11 +15,13 @@ #include <linux/bitops.h> #include <linux/bitmap.h> #include <linux/export.h> -#include <linux/kernel.h> - -#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ - !defined(find_next_and_bit) +#include <linux/math.h> +#include <linux/minmax.h> +#include <linux/swab.h> +#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ + !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) || \ + !defined(find_next_and_bit) /* * This is a common helper function for find_next_bit, find_next_zero_bit, and * find_next_and_bit. The differences are: @@ -27,11 +29,11 @@ * searching it for one bits. * - The optional "addr2", which is anded with "addr1" if present. */ -static inline unsigned long _find_next_bit(const unsigned long *addr1, +unsigned long _find_next_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert) + unsigned long start, unsigned long invert, unsigned long le) { - unsigned long tmp; + unsigned long tmp, mask; if (unlikely(start >= nbits)) return nbits; @@ -42,7 +44,12 @@ static inline unsigned long _find_next_bit(const unsigned long *addr1, tmp ^= invert; /* Handle 1st word. */ - tmp &= BITMAP_FIRST_WORD_MASK(start); + mask = BITMAP_FIRST_WORD_MASK(start); + if (le) + mask = swab(mask); + + tmp &= mask; + start = round_down(start, BITS_PER_LONG); while (!tmp) { @@ -56,46 +63,19 @@ static inline unsigned long _find_next_bit(const unsigned long *addr1, tmp ^= invert; } - return min(start + __ffs(tmp), nbits); -} -#endif - -#ifndef find_next_bit -/* - * Find the next set bit in a memory region. - */ -unsigned long find_next_bit(const unsigned long *addr, unsigned long size, - unsigned long offset) -{ - return _find_next_bit(addr, NULL, size, offset, 0UL); -} -EXPORT_SYMBOL(find_next_bit); -#endif - -#ifndef find_next_zero_bit -unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, - unsigned long offset) -{ - return _find_next_bit(addr, NULL, size, offset, ~0UL); -} -EXPORT_SYMBOL(find_next_zero_bit); -#endif + if (le) + tmp = swab(tmp); -#if !defined(find_next_and_bit) -unsigned long find_next_and_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long size, - unsigned long offset) -{ - return _find_next_bit(addr1, addr2, size, offset, 0UL); + return min(start + __ffs(tmp), nbits); } -EXPORT_SYMBOL(find_next_and_bit); +EXPORT_SYMBOL(_find_next_bit); #endif #ifndef find_first_bit /* * Find the first set bit in a memory region. */ -unsigned long find_first_bit(const unsigned long *addr, unsigned long size) +unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) { unsigned long idx; @@ -106,14 +86,14 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size) return size; } -EXPORT_SYMBOL(find_first_bit); +EXPORT_SYMBOL(_find_first_bit); #endif #ifndef find_first_zero_bit /* * Find the first cleared bit in a memory region. */ -unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) +unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size) { unsigned long idx; @@ -124,11 +104,11 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) return size; } -EXPORT_SYMBOL(find_first_zero_bit); +EXPORT_SYMBOL(_find_first_zero_bit); #endif #ifndef find_last_bit -unsigned long find_last_bit(const unsigned long *addr, unsigned long size) +unsigned long _find_last_bit(const unsigned long *addr, unsigned long size) { if (size) { unsigned long val = BITMAP_LAST_WORD_MASK(size); @@ -144,77 +124,9 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size) } return size; } -EXPORT_SYMBOL(find_last_bit); -#endif - -#ifdef __BIG_ENDIAN - -/* include/linux/byteorder does not support "unsigned long" type */ -static inline unsigned long ext2_swab(const unsigned long y) -{ -#if BITS_PER_LONG == 64 - return (unsigned long) __swab64((u64) y); -#elif BITS_PER_LONG == 32 - return (unsigned long) __swab32((u32) y); -#else -#error BITS_PER_LONG not defined -#endif -} - -#if !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) -static inline unsigned long _find_next_bit_le(const unsigned long *addr1, - const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert) -{ - unsigned long tmp; - - if (unlikely(start >= nbits)) - return nbits; - - tmp = addr1[start / BITS_PER_LONG]; - if (addr2) - tmp &= addr2[start / BITS_PER_LONG]; - tmp ^= invert; - - /* Handle 1st word. */ - tmp &= ext2_swab(BITMAP_FIRST_WORD_MASK(start)); - start = round_down(start, BITS_PER_LONG); - - while (!tmp) { - start += BITS_PER_LONG; - if (start >= nbits) - return nbits; - - tmp = addr1[start / BITS_PER_LONG]; - if (addr2) - tmp &= addr2[start / BITS_PER_LONG]; - tmp ^= invert; - } - - return min(start + __ffs(ext2_swab(tmp)), nbits); -} +EXPORT_SYMBOL(_find_last_bit); #endif -#ifndef find_next_zero_bit_le -unsigned long find_next_zero_bit_le(const void *addr, unsigned - long size, unsigned long offset) -{ - return _find_next_bit_le(addr, NULL, size, offset, ~0UL); -} -EXPORT_SYMBOL(find_next_zero_bit_le); -#endif - -#ifndef find_next_bit_le -unsigned long find_next_bit_le(const void *addr, unsigned - long size, unsigned long offset) -{ - return _find_next_bit_le(addr, NULL, size, offset, 0UL); -} -EXPORT_SYMBOL(find_next_bit_le); -#endif - -#endif /* __BIG_ENDIAN */ - unsigned long find_next_clump8(unsigned long *clump, const unsigned long *addr, unsigned long size, unsigned long offset) { diff --git a/lib/flex_proportions.c b/lib/flex_proportions.c index 7852bfff50b1..53e7eb1dd76c 100644 --- a/lib/flex_proportions.c +++ b/lib/flex_proportions.c @@ -217,11 +217,12 @@ static void fprop_reflect_period_percpu(struct fprop_global *p, } /* Event of type pl happened */ -void __fprop_inc_percpu(struct fprop_global *p, struct fprop_local_percpu *pl) +void __fprop_add_percpu(struct fprop_global *p, struct fprop_local_percpu *pl, + long nr) { fprop_reflect_period_percpu(p, pl); - percpu_counter_add_batch(&pl->events, 1, PROP_BATCH); - percpu_counter_add(&p->events, 1); + percpu_counter_add_batch(&pl->events, nr, PROP_BATCH); + percpu_counter_add(&p->events, nr); } void fprop_fraction_percpu(struct fprop_global *p, @@ -253,21 +254,29 @@ void fprop_fraction_percpu(struct fprop_global *p, } /* - * Like __fprop_inc_percpu() except that event is counted only if the given + * Like __fprop_add_percpu() except that event is counted only if the given * type has fraction smaller than @max_frac/FPROP_FRAC_BASE */ -void __fprop_inc_percpu_max(struct fprop_global *p, - struct fprop_local_percpu *pl, int max_frac) +void __fprop_add_percpu_max(struct fprop_global *p, + struct fprop_local_percpu *pl, int max_frac, long nr) { if (unlikely(max_frac < FPROP_FRAC_BASE)) { unsigned long numerator, denominator; + s64 tmp; fprop_fraction_percpu(p, pl, &numerator, &denominator); - if (numerator > - (((u64)denominator) * max_frac) >> FPROP_FRAC_SHIFT) + /* Adding 'nr' to fraction exceeds max_frac/FPROP_FRAC_BASE? */ + tmp = (u64)denominator * max_frac - + ((u64)numerator << FPROP_FRAC_SHIFT); + if (tmp < 0) { + /* Maximum fraction already exceeded? */ return; - } else - fprop_reflect_period_percpu(p, pl); - percpu_counter_add_batch(&pl->events, 1, PROP_BATCH); - percpu_counter_add(&p->events, 1); + } else if (tmp < nr * (FPROP_FRAC_BASE - max_frac)) { + /* Add just enough for the fraction to saturate */ + nr = div_u64(tmp + FPROP_FRAC_BASE - max_frac - 1, + FPROP_FRAC_BASE - max_frac); + } + } + + __fprop_add_percpu(p, pl, nr); } diff --git a/lib/fonts/Kconfig b/lib/fonts/Kconfig index 37baa79cdd71..c035fde66aeb 100644 --- a/lib/fonts/Kconfig +++ b/lib/fonts/Kconfig @@ -119,6 +119,12 @@ config FONT_TER16x32 This is the high resolution, large version for use with HiDPI screens. If the standard font is unreadable for you, say Y, otherwise say N. +config FONT_6x8 + bool "OLED 6x8 font" if FONTS + depends on FRAMEBUFFER_CONSOLE + help + This font is useful for small displays (OLED). + config FONT_AUTOSELECT def_bool y depends on !FONT_8x8 @@ -132,6 +138,7 @@ config FONT_AUTOSELECT depends on !FONT_SUN12x22 depends on !FONT_10x18 depends on !FONT_TER16x32 + depends on !FONT_6x8 select FONT_8x16 endif # FONT_SUPPORT diff --git a/lib/fonts/Makefile b/lib/fonts/Makefile index ed95070860de..e16f68492174 100644 --- a/lib/fonts/Makefile +++ b/lib/fonts/Makefile @@ -15,6 +15,7 @@ font-objs-$(CONFIG_FONT_ACORN_8x8) += font_acorn_8x8.o font-objs-$(CONFIG_FONT_MINI_4x6) += font_mini_4x6.o font-objs-$(CONFIG_FONT_6x10) += font_6x10.o font-objs-$(CONFIG_FONT_TER16x32) += font_ter16x32.o +font-objs-$(CONFIG_FONT_6x8) += font_6x8.o font-objs += $(font-objs-y) diff --git a/lib/fonts/font_10x18.c b/lib/fonts/font_10x18.c index 532f0ff89a96..5d940db626e7 100644 --- a/lib/fonts/font_10x18.c +++ b/lib/fonts/font_10x18.c @@ -8,8 +8,8 @@ #define FONTDATAMAX 9216 -static const unsigned char fontdata_10x18[FONTDATAMAX] = { - +static const struct font_data fontdata_10x18 = { + { 0, 0, FONTDATAMAX, 0 }, { /* 0 0x00 '^@' */ 0x00, 0x00, /* 0000000000 */ 0x00, 0x00, /* 0000000000 */ @@ -5129,8 +5129,7 @@ static const unsigned char fontdata_10x18[FONTDATAMAX] = { 0x00, 0x00, /* 0000000000 */ 0x00, 0x00, /* 0000000000 */ 0x00, 0x00, /* 0000000000 */ - -}; +} }; const struct font_desc font_10x18 = { @@ -5138,7 +5137,8 @@ const struct font_desc font_10x18 = { .name = "10x18", .width = 10, .height = 18, - .data = fontdata_10x18, + .charcount = 256, + .data = fontdata_10x18.data, #ifdef __sparc__ .pref = 5, #else diff --git a/lib/fonts/font_6x10.c b/lib/fonts/font_6x10.c index 09b2cc03435b..e65df019e0d2 100644 --- a/lib/fonts/font_6x10.c +++ b/lib/fonts/font_6x10.c @@ -1,8 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/font.h> -static const unsigned char fontdata_6x10[] = { +#define FONTDATAMAX 2560 +static const struct font_data fontdata_6x10 = { + { 0, 0, FONTDATAMAX, 0 }, { /* 0 0x00 '^@' */ 0x00, /* 00000000 */ 0x00, /* 00000000 */ @@ -3074,14 +3076,14 @@ static const unsigned char fontdata_6x10[] = { 0x00, /* 00000000 */ 0x00, /* 00000000 */ 0x00, /* 00000000 */ - -}; +} }; const struct font_desc font_6x10 = { .idx = FONT6x10_IDX, .name = "6x10", .width = 6, .height = 10, - .data = fontdata_6x10, + .charcount = 256, + .data = fontdata_6x10.data, .pref = 0, }; diff --git a/lib/fonts/font_6x11.c b/lib/fonts/font_6x11.c index d7136c33f1f0..bd76b3f6b635 100644 --- a/lib/fonts/font_6x11.c +++ b/lib/fonts/font_6x11.c @@ -9,8 +9,8 @@ #define FONTDATAMAX (11*256) -static const unsigned char fontdata_6x11[FONTDATAMAX] = { - +static const struct font_data fontdata_6x11 = { + { 0, 0, FONTDATAMAX, 0 }, { /* 0 0x00 '^@' */ 0x00, /* 00000000 */ 0x00, /* 00000000 */ @@ -3338,8 +3338,7 @@ static const unsigned char fontdata_6x11[FONTDATAMAX] = { 0x00, /* 00000000 */ 0x00, /* 00000000 */ 0x00, /* 00000000 */ - -}; +} }; const struct font_desc font_vga_6x11 = { @@ -3347,7 +3346,8 @@ const struct font_desc font_vga_6x11 = { .name = "ProFont6x11", .width = 6, .height = 11, - .data = fontdata_6x11, + .charcount = 256, + .data = fontdata_6x11.data, /* Try avoiding this font if possible unless on MAC */ .pref = -2000, }; diff --git a/lib/fonts/font_6x8.c b/lib/fonts/font_6x8.c new file mode 100644 index 000000000000..06ace7792521 --- /dev/null +++ b/lib/fonts/font_6x8.c @@ -0,0 +1,2577 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/font.h> + +#define FONTDATAMAX 2048 + +static const struct font_data fontdata_6x8 = { + { 0, 0, FONTDATAMAX, 0 }, { + /* 0 0x00 '^@' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 1 0x01 '^A' */ + 0x78, /* 011110 */ + 0x84, /* 100001 */ + 0xCC, /* 110011 */ + 0x84, /* 100001 */ + 0xCC, /* 110011 */ + 0xB4, /* 101101 */ + 0x78, /* 011110 */ + 0x00, /* 000000 */ + + /* 2 0x02 '^B' */ + 0x78, /* 011110 */ + 0xFC, /* 111111 */ + 0xB4, /* 101101 */ + 0xFC, /* 111111 */ + 0xB4, /* 101101 */ + 0xCC, /* 110011 */ + 0x78, /* 011110 */ + 0x00, /* 000000 */ + + /* 3 0x03 '^C' */ + 0x00, /* 000000 */ + 0x28, /* 001010 */ + 0x7C, /* 011111 */ + 0x7C, /* 011111 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 4 0x04 '^D' */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x7C, /* 011111 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 5 0x05 '^E' */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x38, /* 001110 */ + 0x6C, /* 011011 */ + 0x6C, /* 011011 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 6 0x06 '^F' */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x7C, /* 011111 */ + 0x7C, /* 011111 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 7 0x07 '^G' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x78, /* 011110 */ + 0x30, /* 001100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 8 0x08 '^H' */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xCC, /* 110011 */ + 0x84, /* 100001 */ + 0xCC, /* 110011 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + + /* 9 0x09 '^I' */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x48, /* 010010 */ + 0x84, /* 100001 */ + 0x48, /* 010010 */ + 0x30, /* 001100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 10 0x0A '^J' */ + 0xFC, /* 111111 */ + 0xCC, /* 110011 */ + 0xB4, /* 101101 */ + 0x78, /* 011110 */ + 0xB4, /* 101101 */ + 0xCC, /* 110011 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + + /* 11 0x0B '^K' */ + 0x3C, /* 001111 */ + 0x14, /* 000101 */ + 0x20, /* 001000 */ + 0x78, /* 011110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 12 0x0C '^L' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 13 0x0D '^M' */ + 0x18, /* 000110 */ + 0x14, /* 000101 */ + 0x14, /* 000101 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x70, /* 011100 */ + 0x60, /* 011000 */ + 0x00, /* 000000 */ + + /* 14 0x0E '^N' */ + 0x3C, /* 001111 */ + 0x24, /* 001001 */ + 0x3C, /* 001111 */ + 0x24, /* 001001 */ + 0x24, /* 001001 */ + 0x6C, /* 011011 */ + 0x6C, /* 011011 */ + 0x00, /* 000000 */ + + /* 15 0x0F '^O' */ + 0x10, /* 000100 */ + 0x54, /* 010101 */ + 0x38, /* 001110 */ + 0x6C, /* 011011 */ + 0x38, /* 001110 */ + 0x54, /* 010101 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 16 0x10 '^P' */ + 0x40, /* 010000 */ + 0x60, /* 011000 */ + 0x70, /* 011100 */ + 0x78, /* 011110 */ + 0x70, /* 011100 */ + 0x60, /* 011000 */ + 0x40, /* 010000 */ + 0x00, /* 000000 */ + + /* 17 0x11 '^Q' */ + 0x04, /* 000001 */ + 0x0C, /* 000011 */ + 0x1C, /* 000111 */ + 0x3C, /* 001111 */ + 0x1C, /* 000111 */ + 0x0C, /* 000011 */ + 0x04, /* 000001 */ + 0x00, /* 000000 */ + + /* 18 0x12 '^R' */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x54, /* 010101 */ + 0x10, /* 000100 */ + 0x54, /* 010101 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 19 0x13 '^S' */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x00, /* 000000 */ + 0x48, /* 010010 */ + 0x00, /* 000000 */ + + /* 20 0x14 '^T' */ + 0x3C, /* 001111 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x3C, /* 001111 */ + 0x14, /* 000101 */ + 0x14, /* 000101 */ + 0x14, /* 000101 */ + 0x00, /* 000000 */ + + /* 21 0x15 '^U' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x30, /* 001100 */ + 0x28, /* 001010 */ + 0x14, /* 000101 */ + 0x0C, /* 000011 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + + /* 22 0x16 '^V' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xF8, /* 111110 */ + 0xF8, /* 111110 */ + 0xF8, /* 111110 */ + 0x00, /* 000000 */ + + /* 23 0x17 '^W' */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x54, /* 010101 */ + 0x10, /* 000100 */ + 0x54, /* 010101 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x7C, /* 011111 */ + + /* 24 0x18 '^X' */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x54, /* 010101 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 25 0x19 '^Y' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x54, /* 010101 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 26 0x1A '^Z' */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x7C, /* 011111 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 27 0x1B '^[' */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x7C, /* 011111 */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 28 0x1C '^\' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x78, /* 011110 */ + 0x00, /* 000000 */ + + /* 29 0x1D '^]' */ + 0x00, /* 000000 */ + 0x48, /* 010010 */ + 0x84, /* 100001 */ + 0xFC, /* 111111 */ + 0x84, /* 100001 */ + 0x48, /* 010010 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 30 0x1E '^^' */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x38, /* 001110 */ + 0x7C, /* 011111 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 31 0x1F '^_' */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x7C, /* 011111 */ + 0x38, /* 001110 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 32 0x20 ' ' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 33 0x21 '!' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 34 0x22 '"' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 35 0x23 '#' */ + 0x00, /* 000000 */ + 0x28, /* 001010 */ + 0x7C, /* 011111 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x7C, /* 011111 */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + + /* 36 0x24 '$' */ + 0x10, /* 000000 */ + 0x38, /* 001000 */ + 0x40, /* 010000 */ + 0x30, /* 001000 */ + 0x08, /* 000000 */ + 0x70, /* 011000 */ + 0x20, /* 001000 */ + 0x00, /* 000000 */ + + /* 37 0x25 '%' */ + 0x64, /* 011001 */ + 0x64, /* 011001 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x4C, /* 010011 */ + 0x4C, /* 010011 */ + 0x00, /* 000000 */ + + /* 38 0x26 '&' */ + 0x30, /* 001100 */ + 0x48, /* 010010 */ + 0x50, /* 010100 */ + 0x20, /* 001000 */ + 0x54, /* 010101 */ + 0x48, /* 010010 */ + 0x34, /* 001101 */ + 0x00, /* 000000 */ + + /* 39 0x27 ''' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 40 0x28 '(' */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x20, /* 001000 */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x00, /* 000000 */ + + /* 41 0x29 ')' */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x08, /* 000010 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x00, /* 000000 */ + + /* 42 0x2A '*' */ + 0x10, /* 000100 */ + 0x54, /* 010101 */ + 0x38, /* 001110 */ + 0x54, /* 010101 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 43 0x2B '+' */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x7C, /* 011111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 44 0x2C ',' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x30, /* 001100 */ + 0x20, /* 001000 */ + + /* 45 0x2D '-' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 46 0x2E '.' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x18, /* 000110 */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + + /* 47 0x2F '/' */ + 0x04, /* 000001 */ + 0x08, /* 000010 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x20, /* 001000 */ + 0x40, /* 010000 */ + + /* 48 0x30 '0' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x4C, /* 010011 */ + 0x54, /* 010101 */ + 0x64, /* 011001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 49 0x31 '1' */ + 0x10, /* 000100 */ + 0x30, /* 001100 */ + 0x50, /* 010100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 50 0x32 '2' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x04, /* 000001 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 51 0x33 '3' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x04, /* 000001 */ + 0x18, /* 000110 */ + 0x04, /* 000001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 52 0x34 '4' */ + 0x08, /* 000010 */ + 0x18, /* 000110 */ + 0x28, /* 001010 */ + 0x48, /* 010010 */ + 0x7C, /* 011111 */ + 0x08, /* 000010 */ + 0x08, /* 000010 */ + 0x00, /* 000000 */ + + /* 53 0x35 '5' */ + 0x7C, /* 011111 */ + 0x40, /* 010000 */ + 0x78, /* 011110 */ + 0x04, /* 000001 */ + 0x04, /* 000001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 54 0x36 '6' */ + 0x18, /* 000110 */ + 0x20, /* 001000 */ + 0x40, /* 010000 */ + 0x78, /* 011110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 55 0x37 '7' */ + 0x7C, /* 011111 */ + 0x04, /* 000001 */ + 0x04, /* 000001 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 56 0x38 '8' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 57 0x39 '9' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x3C, /* 001111 */ + 0x04, /* 000001 */ + 0x08, /* 000010 */ + 0x30, /* 001100 */ + 0x00, /* 000000 */ + + /* 58 0x3A ':' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x18, /* 000110 */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x18, /* 000110 */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + + /* 59 0x3B ';' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x30, /* 001100 */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x30, /* 001100 */ + 0x20, /* 001000 */ + + /* 60 0x3C '<' */ + 0x04, /* 000001 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x04, /* 000001 */ + 0x00, /* 000000 */ + + /* 61 0x3D '=' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 62 0x3E '>' */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x04, /* 000001 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x00, /* 000000 */ + + /* 63 0x3F '?' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x04, /* 000001 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 64 0x40 '@' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x5C, /* 010111 */ + 0x54, /* 010101 */ + 0x5C, /* 010111 */ + 0x40, /* 010000 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 65 0x41 'A' */ + 0x10, /* 000100 */ + 0x28, /* 001010 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 66 0x42 'B' */ + 0x78, /* 011110 */ + 0x24, /* 001001 */ + 0x24, /* 001001 */ + 0x38, /* 001110 */ + 0x24, /* 001001 */ + 0x24, /* 001001 */ + 0x78, /* 011110 */ + 0x00, /* 000000 */ + + /* 67 0x43 'C' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 68 0x44 'D' */ + 0x78, /* 011110 */ + 0x24, /* 001001 */ + 0x24, /* 001001 */ + 0x24, /* 001001 */ + 0x24, /* 001001 */ + 0x24, /* 001001 */ + 0x78, /* 011110 */ + 0x00, /* 000000 */ + + /* 69 0x45 'E' */ + 0x7C, /* 011111 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x78, /* 011110 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 70 0x46 'F' */ + 0x7C, /* 011111 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x78, /* 011110 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x00, /* 000000 */ + + /* 71 0x47 'G' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x40, /* 010000 */ + 0x5C, /* 010111 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 72 0x48 'H' */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 73 0x49 'I' */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 74 0x4A 'J' */ + 0x1C, /* 000111 */ + 0x08, /* 000010 */ + 0x08, /* 000010 */ + 0x08, /* 000010 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x30, /* 001100 */ + 0x00, /* 000000 */ + + /* 75 0x4B 'K' */ + 0x44, /* 010001 */ + 0x48, /* 010010 */ + 0x50, /* 010100 */ + 0x60, /* 011000 */ + 0x50, /* 010100 */ + 0x48, /* 010010 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 76 0x4C 'L' */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 77 0x4D 'M' */ + 0x44, /* 010001 */ + 0x6C, /* 011011 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 78 0x4E 'N' */ + 0x44, /* 010001 */ + 0x64, /* 011001 */ + 0x54, /* 010101 */ + 0x4C, /* 010011 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 79 0x4F 'O' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 80 0x50 'P' */ + 0x78, /* 011110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x78, /* 011110 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x00, /* 000000 */ + + /* 81 0x51 'Q' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x54, /* 010101 */ + 0x48, /* 010010 */ + 0x34, /* 001101 */ + 0x00, /* 000000 */ + + /* 82 0x52 'R' */ + 0x78, /* 011110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x78, /* 011110 */ + 0x50, /* 010100 */ + 0x48, /* 010010 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 83 0x53 'S' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x40, /* 010000 */ + 0x38, /* 001110 */ + 0x04, /* 000001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 84 0x54 'T' */ + 0x7C, /* 011111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 85 0x55 'U' */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 86 0x56 'V' */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x28, /* 001010 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 87 0x57 'W' */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x6C, /* 011011 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 88 0x58 'X' */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x28, /* 001010 */ + 0x10, /* 000100 */ + 0x28, /* 001010 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 89 0x59 'Y' */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x28, /* 001010 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 90 0x5A 'Z' */ + 0x7C, /* 011111 */ + 0x04, /* 000001 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x40, /* 010000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 91 0x5B '[' */ + 0x18, /* 000110 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + + /* 92 0x5C '\' */ + 0x40, /* 010000 */ + 0x20, /* 001000 */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x08, /* 000010 */ + 0x04, /* 000001 */ + + /* 93 0x5D ']' */ + 0x30, /* 001100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x30, /* 001100 */ + 0x00, /* 000000 */ + + /* 94 0x5E '^' */ + 0x10, /* 000100 */ + 0x28, /* 001010 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 95 0x5F '_' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + + /* 96 0x60 '`' */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 97 0x61 'a' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x04, /* 000001 */ + 0x3C, /* 001111 */ + 0x44, /* 010001 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 98 0x62 'b' */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x58, /* 010110 */ + 0x64, /* 011001 */ + 0x44, /* 010001 */ + 0x64, /* 011001 */ + 0x58, /* 010110 */ + 0x00, /* 000000 */ + + /* 99 0x63 'c' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x40, /* 010000 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 100 0x64 'd' */ + 0x04, /* 000001 */ + 0x04, /* 000001 */ + 0x34, /* 001101 */ + 0x4C, /* 010011 */ + 0x44, /* 010001 */ + 0x4C, /* 010011 */ + 0x34, /* 001101 */ + 0x00, /* 000000 */ + + /* 101 0x65 'e' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x40, /* 010000 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 102 0x66 'f' */ + 0x0C, /* 000011 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 103 0x67 'g' */ + 0x00, /* 000000 */ + 0x34, /* 001101 */ + 0x4C, /* 010011 */ + 0x44, /* 010001 */ + 0x4C, /* 010011 */ + 0x34, /* 001101 */ + 0x04, /* 000001 */ + 0x38, /* 001110 */ + + /* 104 0x68 'h' */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x78, /* 011110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 105 0x69 'i' */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 106 0x6A 'j' */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x60, /* 011000 */ + + /* 107 0x6B 'k' */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x48, /* 010010 */ + 0x50, /* 010100 */ + 0x70, /* 011100 */ + 0x48, /* 010010 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 108 0x6C 'l' */ + 0x30, /* 001100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 109 0x6D 'm' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x68, /* 011010 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x00, /* 000000 */ + + /* 110 0x6E 'n' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x58, /* 010110 */ + 0x64, /* 011001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 111 0x6F 'o' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 112 0x70 'p' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x78, /* 011110 */ + 0x44, /* 010001 */ + 0x64, /* 011001 */ + 0x58, /* 010110 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + + /* 113 0x71 'q' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x3C, /* 001111 */ + 0x44, /* 010001 */ + 0x4C, /* 010011 */ + 0x34, /* 001101 */ + 0x04, /* 000001 */ + 0x04, /* 000001 */ + + /* 114 0x72 'r' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x58, /* 010110 */ + 0x64, /* 011001 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x00, /* 000000 */ + + /* 115 0x73 's' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x3C, /* 001111 */ + 0x40, /* 010000 */ + 0x38, /* 001110 */ + 0x04, /* 000001 */ + 0x78, /* 011110 */ + 0x00, /* 000000 */ + + /* 116 0x74 't' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x0C, /* 000011 */ + 0x00, /* 000000 */ + + /* 117 0x75 'u' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x4C, /* 010011 */ + 0x34, /* 001101 */ + 0x00, /* 000000 */ + + /* 118 0x76 'v' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x28, /* 001010 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 119 0x77 'w' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + + /* 120 0x78 'x' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x44, /* 010001 */ + 0x28, /* 001010 */ + 0x10, /* 000100 */ + 0x28, /* 001010 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 121 0x79 'y' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x3C, /* 001111 */ + 0x04, /* 000001 */ + 0x38, /* 001110 */ + + /* 122 0x7A 'z' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 123 0x7B '{' */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x00, /* 000000 */ + + /* 124 0x7C '|' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 125 0x7D '}' */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x00, /* 000000 */ + + /* 126 0x7E '~' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x20, /* 001000 */ + 0x54, /* 010101 */ + 0x08, /* 000010 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 127 0x7F '' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x28, /* 001010 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 128 0x80 '\200' */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x40, /* 010000 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + + /* 129 0x81 '\201' */ + 0x00, /* 000000 */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x4C, /* 010011 */ + 0x34, /* 001101 */ + 0x00, /* 000000 */ + + /* 130 0x82 '\202' */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x40, /* 010000 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 131 0x83 '\203' */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x04, /* 000001 */ + 0x3C, /* 001111 */ + 0x44, /* 010001 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 132 0x84 '\204' */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x04, /* 000001 */ + 0x3C, /* 001111 */ + 0x44, /* 010001 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 133 0x85 '\205' */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x04, /* 000001 */ + 0x3C, /* 001111 */ + 0x44, /* 010001 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 134 0x86 '\206' */ + 0x3C, /* 001111 */ + 0x18, /* 000110 */ + 0x38, /* 001110 */ + 0x04, /* 000001 */ + 0x3C, /* 001111 */ + 0x44, /* 010001 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 135 0x87 '\207' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x40, /* 010000 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + + /* 136 0x88 '\210' */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x40, /* 010000 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 137 0x89 '\211' */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x40, /* 010000 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 138 0x8A '\212' */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x40, /* 010000 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 139 0x8B '\213' */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 140 0x8C '\214' */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 141 0x8D '\215' */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 142 0x8E '\216' */ + 0x44, /* 010001 */ + 0x10, /* 000100 */ + 0x28, /* 001010 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 143 0x8F '\217' */ + 0x30, /* 001100 */ + 0x48, /* 010010 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 144 0x90 '\220' */ + 0x10, /* 000100 */ + 0x7C, /* 011111 */ + 0x40, /* 010000 */ + 0x78, /* 011110 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 145 0x91 '\221' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x78, /* 011110 */ + 0x14, /* 000101 */ + 0x7C, /* 011111 */ + 0x50, /* 010100 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 146 0x92 '\222' */ + 0x3C, /* 001111 */ + 0x50, /* 010100 */ + 0x50, /* 010100 */ + 0x78, /* 011110 */ + 0x50, /* 010100 */ + 0x50, /* 010100 */ + 0x5C, /* 010111 */ + 0x00, /* 000000 */ + + /* 147 0x93 '\223' */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 148 0x94 '\224' */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 149 0x95 '\225' */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 150 0x96 '\226' */ + 0x10, /* 000100 */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x4C, /* 010011 */ + 0x34, /* 001101 */ + 0x00, /* 000000 */ + + /* 151 0x97 '\227' */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x4C, /* 010011 */ + 0x34, /* 001101 */ + 0x00, /* 000000 */ + + /* 152 0x98 '\230' */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x3C, /* 001111 */ + 0x04, /* 000001 */ + 0x38, /* 001110 */ + + /* 153 0x99 '\231' */ + 0x84, /* 100001 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 154 0x9A '\232' */ + 0x88, /* 100010 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 155 0x9B '\233' */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x54, /* 010101 */ + 0x50, /* 010100 */ + 0x54, /* 010101 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 156 0x9C '\234' */ + 0x30, /* 001100 */ + 0x48, /* 010010 */ + 0x40, /* 010000 */ + 0x70, /* 011100 */ + 0x40, /* 010000 */ + 0x44, /* 010001 */ + 0x78, /* 011110 */ + 0x00, /* 000000 */ + + /* 157 0x9D '\235' */ + 0x44, /* 010001 */ + 0x28, /* 001010 */ + 0x7C, /* 011111 */ + 0x10, /* 000100 */ + 0x7C, /* 011111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 158 0x9E '\236' */ + 0x70, /* 011100 */ + 0x48, /* 010010 */ + 0x70, /* 011100 */ + 0x48, /* 010010 */ + 0x5C, /* 010111 */ + 0x48, /* 010010 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 159 0x9F '\237' */ + 0x0C, /* 000011 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x60, /* 011000 */ + 0x00, /* 000000 */ + + /* 160 0xA0 '\240' */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x04, /* 000001 */ + 0x3C, /* 001111 */ + 0x44, /* 010001 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 161 0xA1 '\241' */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x30, /* 001100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 162 0xA2 '\242' */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 163 0xA3 '\243' */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x4C, /* 010011 */ + 0x34, /* 001101 */ + 0x00, /* 000000 */ + + /* 164 0xA4 '\244' */ + 0x34, /* 001101 */ + 0x58, /* 010110 */ + 0x00, /* 000000 */ + 0x58, /* 010110 */ + 0x64, /* 011001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 165 0xA5 '\245' */ + 0x58, /* 010110 */ + 0x44, /* 010001 */ + 0x64, /* 011001 */ + 0x54, /* 010101 */ + 0x4C, /* 010011 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 166 0xA6 '\246' */ + 0x38, /* 001110 */ + 0x04, /* 000001 */ + 0x3C, /* 001111 */ + 0x44, /* 010001 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 167 0xA7 '\247' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 168 0xA8 '\250' */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x40, /* 010000 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 169 0xA9 '\251' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 170 0xAA '\252' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x04, /* 000001 */ + 0x04, /* 000001 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 171 0xAB '\253' */ + 0x20, /* 001000 */ + 0x24, /* 001001 */ + 0x28, /* 001010 */ + 0x10, /* 000100 */ + 0x28, /* 001010 */ + 0x44, /* 010001 */ + 0x08, /* 000010 */ + 0x1C, /* 000111 */ + + /* 172 0xAC '\254' */ + 0x20, /* 001000 */ + 0x24, /* 001001 */ + 0x28, /* 001010 */ + 0x10, /* 000100 */ + 0x28, /* 001010 */ + 0x58, /* 010110 */ + 0x3C, /* 001111 */ + 0x08, /* 000010 */ + + /* 173 0xAD '\255' */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + + /* 174 0xAE '\256' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x24, /* 001001 */ + 0x48, /* 010010 */ + 0x90, /* 100100 */ + 0x48, /* 010010 */ + 0x24, /* 001001 */ + 0x00, /* 000000 */ + + /* 175 0xAF '\257' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x90, /* 100100 */ + 0x48, /* 010010 */ + 0x24, /* 001001 */ + 0x48, /* 010010 */ + 0x90, /* 100100 */ + 0x00, /* 000000 */ + + /* 176 0xB0 '\260' */ + 0x10, /* 000100 */ + 0x44, /* 010001 */ + 0x10, /* 000100 */ + 0x44, /* 010001 */ + 0x10, /* 000100 */ + 0x44, /* 010001 */ + 0x10, /* 000100 */ + 0x44, /* 010001 */ + + /* 177 0xB1 '\261' */ + 0xA8, /* 101010 */ + 0x54, /* 010101 */ + 0xA8, /* 101010 */ + 0x54, /* 010101 */ + 0xA8, /* 101010 */ + 0x54, /* 010101 */ + 0xA8, /* 101010 */ + 0x54, /* 010101 */ + + /* 178 0xB2 '\262' */ + 0xDC, /* 110111 */ + 0x74, /* 011101 */ + 0xDC, /* 110111 */ + 0x74, /* 011101 */ + 0xDC, /* 110111 */ + 0x74, /* 011101 */ + 0xDC, /* 110111 */ + 0x74, /* 011101 */ + + /* 179 0xB3 '\263' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 180 0xB4 '\264' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0xF0, /* 111100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 181 0xB5 '\265' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0xF0, /* 111100 */ + 0x10, /* 000100 */ + 0xF0, /* 111100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 182 0xB6 '\266' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0xE8, /* 111010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 183 0xB7 '\267' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xF8, /* 111110 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 184 0xB8 '\270' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xF0, /* 111100 */ + 0x10, /* 000100 */ + 0xF0, /* 111100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 185 0xB9 '\271' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0xE8, /* 111010 */ + 0x08, /* 000010 */ + 0xE8, /* 111010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 186 0xBA '\272' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 187 0xBB '\273' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xF8, /* 111110 */ + 0x08, /* 000010 */ + 0xE8, /* 111010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 188 0xBC '\274' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0xE8, /* 111010 */ + 0x08, /* 000010 */ + 0xF8, /* 111110 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 189 0xBD '\275' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0xF8, /* 111110 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 190 0xBE '\276' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0xF0, /* 111100 */ + 0x10, /* 000100 */ + 0xF0, /* 111100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 191 0xBF '\277' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xF0, /* 111100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 192 0xC0 '\300' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x1C, /* 000111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 193 0xC1 '\301' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 194 0xC2 '\302' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 195 0xC3 '\303' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x1C, /* 000111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 196 0xC4 '\304' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 197 0xC5 '\305' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0xFC, /* 111111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 198 0xC6 '\306' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x1C, /* 000111 */ + 0x10, /* 000100 */ + 0x1C, /* 000111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 199 0xC7 '\307' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x2C, /* 001011 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 200 0xC8 '\310' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x2C, /* 001011 */ + 0x20, /* 001000 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 201 0xC9 '\311' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x3C, /* 001111 */ + 0x20, /* 001000 */ + 0x2C, /* 001011 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 202 0xCA '\312' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0xEC, /* 111011 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 203 0xCB '\313' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0xEC, /* 111011 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 204 0xCC '\314' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x2C, /* 001011 */ + 0x20, /* 001000 */ + 0x2C, /* 001011 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 205 0xCD '\315' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 206 0xCE '\316' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0xEC, /* 111011 */ + 0x00, /* 000000 */ + 0xEC, /* 111011 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 207 0xCF '\317' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 208 0xD0 '\320' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 209 0xD1 '\321' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 210 0xD2 '\322' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 211 0xD3 '\323' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 212 0xD4 '\324' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x1C, /* 000111 */ + 0x10, /* 000100 */ + 0x1C, /* 000111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 213 0xD5 '\325' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x1C, /* 000111 */ + 0x10, /* 000100 */ + 0x1C, /* 000111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 214 0xD6 '\326' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x3C, /* 001111 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 215 0xD7 '\327' */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0xFC, /* 111111 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + + /* 216 0xD8 '\330' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0xFC, /* 111111 */ + 0x10, /* 000100 */ + 0xFC, /* 111111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 217 0xD9 '\331' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0xF0, /* 111100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 218 0xDA '\332' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x1C, /* 000111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 219 0xDB '\333' */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + + /* 220 0xDC '\334' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + + /* 221 0xDD '\335' */ + 0xE0, /* 111000 */ + 0xE0, /* 111000 */ + 0xE0, /* 111000 */ + 0xE0, /* 111000 */ + 0xE0, /* 111000 */ + 0xE0, /* 111000 */ + 0xE0, /* 111000 */ + 0xE0, /* 111000 */ + + /* 222 0xDE '\336' */ + 0x1C, /* 000111 */ + 0x1C, /* 000111 */ + 0x1C, /* 000111 */ + 0x1C, /* 000111 */ + 0x1C, /* 000111 */ + 0x1C, /* 000111 */ + 0x1C, /* 000111 */ + 0x1C, /* 000111 */ + + /* 223 0xDF '\337' */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 224 0xE0 '\340' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x34, /* 001101 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x34, /* 001101 */ + 0x00, /* 000000 */ + + /* 225 0xE1 '\341' */ + 0x24, /* 001001 */ + 0x44, /* 010001 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x58, /* 010110 */ + 0x40, /* 010000 */ + + /* 226 0xE2 '\342' */ + 0x7C, /* 011111 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x00, /* 000000 */ + + /* 227 0xE3 '\343' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x28, /* 001010 */ + 0x00, /* 000000 */ + + /* 228 0xE4 '\344' */ + 0x7C, /* 011111 */ + 0x24, /* 001001 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x24, /* 001001 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 229 0xE5 '\345' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x3C, /* 001111 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x30, /* 001100 */ + 0x00, /* 000000 */ + + /* 230 0xE6 '\346' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x74, /* 011101 */ + 0x40, /* 010000 */ + + /* 231 0xE7 '\347' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x0C, /* 000011 */ + 0x00, /* 000000 */ + + /* 232 0xE8 '\350' */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 233 0xE9 '\351' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x7C, /* 011111 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 234 0xEA '\352' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x28, /* 001010 */ + 0x6C, /* 011011 */ + 0x00, /* 000000 */ + + /* 235 0xEB '\353' */ + 0x18, /* 000110 */ + 0x20, /* 001000 */ + 0x18, /* 000110 */ + 0x24, /* 001001 */ + 0x24, /* 001001 */ + 0x24, /* 001001 */ + 0x18, /* 000110 */ + 0x00, /* 000000 */ + + /* 236 0xEC '\354' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 237 0xED '\355' */ + 0x00, /* 000000 */ + 0x04, /* 000001 */ + 0x38, /* 001110 */ + 0x54, /* 010101 */ + 0x54, /* 010101 */ + 0x38, /* 001110 */ + 0x40, /* 010000 */ + 0x00, /* 000000 */ + + /* 238 0xEE '\356' */ + 0x3C, /* 001111 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x38, /* 001110 */ + 0x40, /* 010000 */ + 0x40, /* 010000 */ + 0x3C, /* 001111 */ + 0x00, /* 000000 */ + + /* 239 0xEF '\357' */ + 0x38, /* 001110 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x44, /* 010001 */ + 0x00, /* 000000 */ + + /* 240 0xF0 '\360' */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0xFC, /* 111111 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 241 0xF1 '\361' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x7C, /* 011111 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + + /* 242 0xF2 '\362' */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 243 0xF3 '\363' */ + 0x08, /* 000010 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x10, /* 000100 */ + 0x08, /* 000010 */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 244 0xF4 '\364' */ + 0x0C, /* 000011 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + + /* 245 0xF5 '\365' */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x10, /* 000100 */ + 0x60, /* 011000 */ + + /* 246 0xF6 '\366' */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x7C, /* 011111 */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 247 0xF7 '\367' */ + 0x00, /* 000000 */ + 0x20, /* 001000 */ + 0x54, /* 010101 */ + 0x08, /* 000010 */ + 0x20, /* 001000 */ + 0x54, /* 010101 */ + 0x08, /* 000010 */ + 0x00, /* 000000 */ + + /* 248 0xF8 '\370' */ + 0x30, /* 001100 */ + 0x48, /* 010010 */ + 0x48, /* 010010 */ + 0x30, /* 001100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 249 0xF9 '\371' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x38, /* 001110 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 250 0xFA '\372' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x10, /* 000100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 251 0xFB '\373' */ + 0x04, /* 000001 */ + 0x08, /* 000010 */ + 0x08, /* 000010 */ + 0x50, /* 010100 */ + 0x50, /* 010100 */ + 0x20, /* 001000 */ + 0x20, /* 001000 */ + 0x00, /* 000000 */ + + /* 252 0xFC '\374' */ + 0x60, /* 011000 */ + 0x50, /* 010100 */ + 0x50, /* 010100 */ + 0x50, /* 010100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 253 0xFD '\375' */ + 0x60, /* 011000 */ + 0x10, /* 000100 */ + 0x20, /* 001000 */ + 0x70, /* 011100 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + + /* 254 0xFE '\376' */ + 0x00, /* 000000 */ + 0x38, /* 001110 */ + 0x38, /* 001110 */ + 0x38, /* 001110 */ + 0x38, /* 001110 */ + 0x38, /* 001110 */ + 0x38, /* 001110 */ + 0x00, /* 000000 */ + + /* 255 0xFF '\377' */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ + 0x00, /* 000000 */ +} }; + +const struct font_desc font_6x8 = { + .idx = FONT6x8_IDX, + .name = "6x8", + .width = 6, + .height = 8, + .charcount = 256, + .data = fontdata_6x8.data, + .pref = 0, +}; diff --git a/lib/fonts/font_7x14.c b/lib/fonts/font_7x14.c index 89752d0b23e8..a2f561c9fa04 100644 --- a/lib/fonts/font_7x14.c +++ b/lib/fonts/font_7x14.c @@ -8,8 +8,8 @@ #define FONTDATAMAX 3584 -static const unsigned char fontdata_7x14[FONTDATAMAX] = { - +static const struct font_data fontdata_7x14 = { + { 0, 0, FONTDATAMAX, 0 }, { /* 0 0x00 '^@' */ 0x00, /* 0000000 */ 0x00, /* 0000000 */ @@ -4105,8 +4105,7 @@ static const unsigned char fontdata_7x14[FONTDATAMAX] = { 0x00, /* 0000000 */ 0x00, /* 0000000 */ 0x00, /* 0000000 */ - -}; +} }; const struct font_desc font_7x14 = { @@ -4114,6 +4113,7 @@ const struct font_desc font_7x14 = { .name = "7x14", .width = 7, .height = 14, - .data = fontdata_7x14, + .charcount = 256, + .data = fontdata_7x14.data, .pref = 0, }; diff --git a/lib/fonts/font_8x16.c b/lib/fonts/font_8x16.c index b7ab1f5fbdb8..06ae14088514 100644 --- a/lib/fonts/font_8x16.c +++ b/lib/fonts/font_8x16.c @@ -10,8 +10,8 @@ #define FONTDATAMAX 4096 -static const unsigned char fontdata_8x16[FONTDATAMAX] = { - +static const struct font_data fontdata_8x16 = { + { 0, 0, FONTDATAMAX, 0 }, { /* 0 0x00 '^@' */ 0x00, /* 00000000 */ 0x00, /* 00000000 */ @@ -4619,8 +4619,7 @@ static const unsigned char fontdata_8x16[FONTDATAMAX] = { 0x00, /* 00000000 */ 0x00, /* 00000000 */ 0x00, /* 00000000 */ - -}; +} }; const struct font_desc font_vga_8x16 = { @@ -4628,7 +4627,8 @@ const struct font_desc font_vga_8x16 = { .name = "VGA8x16", .width = 8, .height = 16, - .data = fontdata_8x16, + .charcount = 256, + .data = fontdata_8x16.data, .pref = 0, }; EXPORT_SYMBOL(font_vga_8x16); diff --git a/lib/fonts/font_8x8.c b/lib/fonts/font_8x8.c index 2328ebc8bab5..69570b8c31af 100644 --- a/lib/fonts/font_8x8.c +++ b/lib/fonts/font_8x8.c @@ -9,8 +9,8 @@ #define FONTDATAMAX 2048 -static const unsigned char fontdata_8x8[FONTDATAMAX] = { - +static const struct font_data fontdata_8x8 = { + { 0, 0, FONTDATAMAX, 0 }, { /* 0 0x00 '^@' */ 0x00, /* 00000000 */ 0x00, /* 00000000 */ @@ -2570,8 +2570,7 @@ static const unsigned char fontdata_8x8[FONTDATAMAX] = { 0x00, /* 00000000 */ 0x00, /* 00000000 */ 0x00, /* 00000000 */ - -}; +} }; const struct font_desc font_vga_8x8 = { @@ -2579,6 +2578,7 @@ const struct font_desc font_vga_8x8 = { .name = "VGA8x8", .width = 8, .height = 8, - .data = fontdata_8x8, + .charcount = 256, + .data = fontdata_8x8.data, .pref = 0, }; diff --git a/lib/fonts/font_acorn_8x8.c b/lib/fonts/font_acorn_8x8.c index 0ff0e85d4481..18755c33d249 100644 --- a/lib/fonts/font_acorn_8x8.c +++ b/lib/fonts/font_acorn_8x8.c @@ -3,7 +3,10 @@ #include <linux/font.h> -static const unsigned char acorndata_8x8[] = { +#define FONTDATAMAX 2048 + +static const struct font_data acorndata_8x8 = { +{ 0, 0, FONTDATAMAX, 0 }, { /* 00 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* ^@ */ /* 01 */ 0x7e, 0x81, 0xa5, 0x81, 0xbd, 0x99, 0x81, 0x7e, /* ^A */ /* 02 */ 0x7e, 0xff, 0xbd, 0xff, 0xc3, 0xe7, 0xff, 0x7e, /* ^B */ @@ -260,14 +263,15 @@ static const unsigned char acorndata_8x8[] = { /* FD */ 0x38, 0x04, 0x18, 0x20, 0x3c, 0x00, 0x00, 0x00, /* FE */ 0x00, 0x00, 0x3c, 0x3c, 0x3c, 0x3c, 0x00, 0x00, /* FF */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; +} }; const struct font_desc font_acorn_8x8 = { .idx = ACORN8x8_IDX, .name = "Acorn8x8", .width = 8, .height = 8, - .data = acorndata_8x8, + .charcount = 256, + .data = acorndata_8x8.data, #ifdef CONFIG_ARCH_ACORN .pref = 20, #else diff --git a/lib/fonts/font_mini_4x6.c b/lib/fonts/font_mini_4x6.c index 838caa1cfef7..8d39fd447952 100644 --- a/lib/fonts/font_mini_4x6.c +++ b/lib/fonts/font_mini_4x6.c @@ -43,8 +43,8 @@ __END__; #define FONTDATAMAX 1536 -static const unsigned char fontdata_mini_4x6[FONTDATAMAX] = { - +static const struct font_data fontdata_mini_4x6 = { + { 0, 0, FONTDATAMAX, 0 }, { /*{*/ /* Char 0: ' ' */ 0xee, /*= [*** ] */ @@ -2145,14 +2145,15 @@ static const unsigned char fontdata_mini_4x6[FONTDATAMAX] = { 0xee, /*= [*** ] */ 0x00, /*= [ ] */ /*}*/ -}; +} }; const struct font_desc font_mini_4x6 = { .idx = MINI4x6_IDX, .name = "MINI4x6", .width = 4, .height = 6, - .data = fontdata_mini_4x6, + .charcount = 256, + .data = fontdata_mini_4x6.data, .pref = 3, }; diff --git a/lib/fonts/font_pearl_8x8.c b/lib/fonts/font_pearl_8x8.c index b15d3c342c5b..ae98ca17982e 100644 --- a/lib/fonts/font_pearl_8x8.c +++ b/lib/fonts/font_pearl_8x8.c @@ -3,7 +3,7 @@ /* */ /* Font file generated by cpi2fnt */ /* ------------------------------ */ -/* Combined with the alpha-numeric */ +/* Combined with the alphanumeric */ /* portion of Greg Harp's old PEARL */ /* font (from earlier versions of */ /* linux-m86k) by John Shifflett */ @@ -14,8 +14,8 @@ #define FONTDATAMAX 2048 -static const unsigned char fontdata_pearl8x8[FONTDATAMAX] = { - +static const struct font_data fontdata_pearl8x8 = { + { 0, 0, FONTDATAMAX, 0 }, { /* 0 0x00 '^@' */ 0x00, /* 00000000 */ 0x00, /* 00000000 */ @@ -2575,14 +2575,14 @@ static const unsigned char fontdata_pearl8x8[FONTDATAMAX] = { 0x00, /* 00000000 */ 0x00, /* 00000000 */ 0x00, /* 00000000 */ - -}; +} }; const struct font_desc font_pearl_8x8 = { .idx = PEARL8x8_IDX, .name = "PEARL8x8", .width = 8, .height = 8, - .data = fontdata_pearl8x8, + .charcount = 256, + .data = fontdata_pearl8x8.data, .pref = 2, }; diff --git a/lib/fonts/font_sun12x22.c b/lib/fonts/font_sun12x22.c index 955d6eee3959..91daf5ab8b6b 100644 --- a/lib/fonts/font_sun12x22.c +++ b/lib/fonts/font_sun12x22.c @@ -3,8 +3,8 @@ #define FONTDATAMAX 11264 -static const unsigned char fontdata_sun12x22[FONTDATAMAX] = { - +static const struct font_data fontdata_sun12x22 = { + { 0, 0, FONTDATAMAX, 0 }, { /* 0 0x00 '^@' */ 0x00, 0x00, /* 000000000000 */ 0x00, 0x00, /* 000000000000 */ @@ -6148,8 +6148,7 @@ static const unsigned char fontdata_sun12x22[FONTDATAMAX] = { 0x00, 0x00, /* 000000000000 */ 0x00, 0x00, /* 000000000000 */ 0x00, 0x00, /* 000000000000 */ - -}; +} }; const struct font_desc font_sun_12x22 = { @@ -6157,7 +6156,8 @@ const struct font_desc font_sun_12x22 = { .name = "SUN12x22", .width = 12, .height = 22, - .data = fontdata_sun12x22, + .charcount = 256, + .data = fontdata_sun12x22.data, #ifdef __sparc__ .pref = 5, #else diff --git a/lib/fonts/font_sun8x16.c b/lib/fonts/font_sun8x16.c index 03d71e53954a..81bb4eeae04e 100644 --- a/lib/fonts/font_sun8x16.c +++ b/lib/fonts/font_sun8x16.c @@ -3,7 +3,8 @@ #define FONTDATAMAX 4096 -static const unsigned char fontdata_sun8x16[FONTDATAMAX] = { +static const struct font_data fontdata_sun8x16 = { +{ 0, 0, FONTDATAMAX, 0 }, { /* */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* */ 0x00,0x00,0x7e,0x81,0xa5,0x81,0x81,0xbd,0x99,0x81,0x81,0x7e,0x00,0x00,0x00,0x00, /* */ 0x00,0x00,0x7e,0xff,0xdb,0xff,0xff,0xc3,0xe7,0xff,0xff,0x7e,0x00,0x00,0x00,0x00, @@ -260,14 +261,15 @@ static const unsigned char fontdata_sun8x16[FONTDATAMAX] = { /* */ 0x00,0x70,0xd8,0x30,0x60,0xc8,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* */ 0x00,0x00,0x00,0x00,0x7c,0x7c,0x7c,0x7c,0x7c,0x7c,0x7c,0x00,0x00,0x00,0x00,0x00, /* */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -}; +} }; const struct font_desc font_sun_8x16 = { .idx = SUN8x16_IDX, .name = "SUN8x16", .width = 8, .height = 16, - .data = fontdata_sun8x16, + .charcount = 256, + .data = fontdata_sun8x16.data, #ifdef __sparc__ .pref = 10, #else diff --git a/lib/fonts/font_ter16x32.c b/lib/fonts/font_ter16x32.c index 3f0cf1ccdf3a..5baedc573dd6 100644 --- a/lib/fonts/font_ter16x32.c +++ b/lib/fonts/font_ter16x32.c @@ -4,8 +4,8 @@ #define FONTDATAMAX 16384 -static const unsigned char fontdata_ter16x32[FONTDATAMAX] = { - +static const struct font_data fontdata_ter16x32 = { + { 0, 0, FONTDATAMAX, 0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, @@ -774,8 +774,8 @@ static const unsigned char fontdata_ter16x32[FONTDATAMAX] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 95 */ - 0x00, 0x00, 0x1c, 0x00, 0x0e, 0x00, 0x07, 0x00, - 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1169,7 +1169,7 @@ static const unsigned char fontdata_ter16x32[FONTDATAMAX] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7f, 0xf8, 0x7f, 0xfc, 0x03, 0x9e, 0x03, 0x8e, + 0x7e, 0xf8, 0x7f, 0xfc, 0x03, 0x9e, 0x03, 0x8e, 0x03, 0x8e, 0x3f, 0x8e, 0x7f, 0xfe, 0xf3, 0xfe, 0xe3, 0x80, 0xe3, 0x80, 0xe3, 0x80, 0xf3, 0xce, 0x7f, 0xfe, 0x3e, 0xfc, 0x00, 0x00, 0x00, 0x00, @@ -2054,8 +2054,7 @@ static const unsigned char fontdata_ter16x32[FONTDATAMAX] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 255 */ - -}; +} }; const struct font_desc font_ter_16x32 = { @@ -2063,7 +2062,8 @@ const struct font_desc font_ter_16x32 = { .name = "TER16x32", .width = 16, .height = 32, - .data = fontdata_ter16x32, + .charcount = 256, + .data = fontdata_ter16x32.data, #ifdef __sparc__ .pref = 5, #else diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c index e7258d8c252b..5f4b07b56cd9 100644 --- a/lib/fonts/fonts.c +++ b/lib/fonts/fonts.c @@ -57,6 +57,9 @@ static const struct font_desc *fonts[] = { #ifdef CONFIG_FONT_TER16x32 &font_ter_16x32, #endif +#ifdef CONFIG_FONT_6x8 + &font_6x8, +#endif }; #define num_fonts ARRAY_SIZE(fonts) diff --git a/lib/genalloc.c b/lib/genalloc.c index 7f1244b5294a..9a57257988c7 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -81,14 +81,15 @@ static int clear_bits_ll(unsigned long *addr, unsigned long mask_to_clear) * users set the same bit, one user will return remain bits, otherwise * return 0. */ -static int bitmap_set_ll(unsigned long *map, int start, int nr) +static unsigned long +bitmap_set_ll(unsigned long *map, unsigned long start, unsigned long nr) { unsigned long *p = map + BIT_WORD(start); - const int size = start + nr; + const unsigned long size = start + nr; int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG); unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start); - while (nr - bits_to_set >= 0) { + while (nr >= bits_to_set) { if (set_bits_ll(p, mask_to_set)) return nr; nr -= bits_to_set; @@ -116,14 +117,15 @@ static int bitmap_set_ll(unsigned long *map, int start, int nr) * users clear the same bit, one user will return remain bits, * otherwise return 0. */ -static int bitmap_clear_ll(unsigned long *map, int start, int nr) +static unsigned long +bitmap_clear_ll(unsigned long *map, unsigned long start, unsigned long nr) { unsigned long *p = map + BIT_WORD(start); - const int size = start + nr; + const unsigned long size = start + nr; int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); - while (nr - bits_to_clear >= 0) { + while (nr >= bits_to_clear) { if (clear_bits_ll(p, mask_to_clear)) return nr; nr -= bits_to_clear; @@ -183,8 +185,8 @@ int gen_pool_add_owner(struct gen_pool *pool, unsigned long virt, phys_addr_t ph size_t size, int nid, void *owner) { struct gen_pool_chunk *chunk; - int nbits = size >> pool->min_alloc_order; - int nbytes = sizeof(struct gen_pool_chunk) + + unsigned long nbits = size >> pool->min_alloc_order; + unsigned long nbytes = sizeof(struct gen_pool_chunk) + BITS_TO_LONGS(nbits) * sizeof(long); chunk = vzalloc_node(nbytes, nid); @@ -242,7 +244,7 @@ void gen_pool_destroy(struct gen_pool *pool) struct list_head *_chunk, *_next_chunk; struct gen_pool_chunk *chunk; int order = pool->min_alloc_order; - int bit, end_bit; + unsigned long bit, end_bit; list_for_each_safe(_chunk, _next_chunk, &pool->chunks) { chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk); @@ -278,7 +280,7 @@ unsigned long gen_pool_alloc_algo_owner(struct gen_pool *pool, size_t size, struct gen_pool_chunk *chunk; unsigned long addr = 0; int order = pool->min_alloc_order; - int nbits, start_bit, end_bit, remain; + unsigned long nbits, start_bit, end_bit, remain; #ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG BUG_ON(in_nmi()); @@ -487,7 +489,7 @@ void gen_pool_free_owner(struct gen_pool *pool, unsigned long addr, size_t size, { struct gen_pool_chunk *chunk; int order = pool->min_alloc_order; - int start_bit, nbits, remain; + unsigned long start_bit, nbits, remain; #ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG BUG_ON(in_nmi()); @@ -640,6 +642,7 @@ EXPORT_SYMBOL(gen_pool_set_algo); * @nr: The number of zeroed bits we're looking for * @data: additional data - unused * @pool: pool to find the fit region memory from + * @start_addr: not used in this function */ unsigned long gen_pool_first_fit(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, void *data, @@ -658,6 +661,7 @@ EXPORT_SYMBOL(gen_pool_first_fit); * @nr: The number of zeroed bits we're looking for * @data: data for alignment * @pool: pool to get order from + * @start_addr: start addr of alloction chunk */ unsigned long gen_pool_first_fit_align(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, void *data, @@ -685,6 +689,7 @@ EXPORT_SYMBOL(gen_pool_first_fit_align); * @nr: The number of zeroed bits we're looking for * @data: data for alignment * @pool: pool to get order from + * @start_addr: not used in this function */ unsigned long gen_pool_fixed_alloc(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, void *data, @@ -719,6 +724,7 @@ EXPORT_SYMBOL(gen_pool_fixed_alloc); * @nr: The number of zeroed bits we're looking for * @data: additional data - unused * @pool: pool to find the fit region memory from + * @start_addr: not used in this function */ unsigned long gen_pool_first_fit_order_align(unsigned long *map, unsigned long size, unsigned long start, @@ -733,13 +739,14 @@ EXPORT_SYMBOL(gen_pool_first_fit_order_align); /** * gen_pool_best_fit - find the best fitting region of memory - * macthing the size requirement (no alignment constraint) + * matching the size requirement (no alignment constraint) * @map: The address to base the search on * @size: The bitmap size in bits * @start: The bitnumber to start searching at * @nr: The number of zeroed bits we're looking for * @data: additional data - unused * @pool: pool to find the fit region memory from + * @start_addr: not used in this function * * Iterate over the bitmap to find the smallest free region * which we can allocate the memory. @@ -755,7 +762,7 @@ unsigned long gen_pool_best_fit(unsigned long *map, unsigned long size, index = bitmap_find_next_zero_area(map, size, start, nr, 0); while (index < size) { - int next_bit = find_next_bit(map, size, index + nr); + unsigned long next_bit = find_next_bit(map, size, index + nr); if ((next_bit - index) < len) { len = next_bit - index; start_bit = index; diff --git a/lib/glob.c b/lib/glob.c index 0ba3ea86b546..85ecbda45cd8 100644 --- a/lib/glob.c +++ b/lib/glob.c @@ -102,7 +102,7 @@ bool __pure glob_match(char const *pat, char const *str) break; case '\\': d = *pat++; - /*FALLTHROUGH*/ + fallthrough; default: /* Literal character */ literal: if (c == d) { diff --git a/lib/hexdump.c b/lib/hexdump.c index 147133f8eb2f..9301578f98e8 100644 --- a/lib/hexdump.c +++ b/lib/hexdump.c @@ -7,6 +7,7 @@ #include <linux/ctype.h> #include <linux/errno.h> #include <linux/kernel.h> +#include <linux/minmax.h> #include <linux/export.h> #include <asm/unaligned.h> diff --git a/lib/idr.c b/lib/idr.c index c2cf2c52bbde..f4ab4f4aa3c7 100644 --- a/lib/idr.c +++ b/lib/idr.c @@ -372,7 +372,8 @@ EXPORT_SYMBOL(idr_replace); * Allocate an ID between @min and @max, inclusive. The allocated ID will * not exceed %INT_MAX, even if @max is larger. * - * Context: Any context. + * Context: Any context. It is safe to call this function without + * locking in your code. * Return: The allocated ID, or %-ENOMEM if memory could not be allocated, * or %-ENOSPC if there are no free IDs. */ @@ -470,6 +471,7 @@ alloc: goto retry; nospc: xas_unlock_irqrestore(&xas, flags); + kfree(alloc); return -ENOSPC; } EXPORT_SYMBOL(ida_alloc_range); @@ -479,7 +481,8 @@ EXPORT_SYMBOL(ida_alloc_range); * @ida: IDA handle. * @id: Previously allocated ID. * - * Context: Any context. + * Context: Any context. It is safe to call this function without + * locking in your code. */ void ida_free(struct ida *ida, unsigned int id) { @@ -531,7 +534,8 @@ EXPORT_SYMBOL(ida_free); * or freed. If the IDA is already empty, there is no need to call this * function. * - * Context: Any context. + * Context: Any context. It is safe to call this function without + * locking in your code. */ void ida_destroy(struct ida *ida) { diff --git a/lib/iomap.c b/lib/iomap.c index e909ab71e995..fbaa3e8f19d6 100644 --- a/lib/iomap.c +++ b/lib/iomap.c @@ -70,27 +70,27 @@ static void bad_io_access(unsigned long port, const char *access) #define mmio_read64be(addr) swab64(readq(addr)) #endif -unsigned int ioread8(void __iomem *addr) +unsigned int ioread8(const void __iomem *addr) { IO_COND(addr, return inb(port), return readb(addr)); return 0xff; } -unsigned int ioread16(void __iomem *addr) +unsigned int ioread16(const void __iomem *addr) { IO_COND(addr, return inw(port), return readw(addr)); return 0xffff; } -unsigned int ioread16be(void __iomem *addr) +unsigned int ioread16be(const void __iomem *addr) { IO_COND(addr, return pio_read16be(port), return mmio_read16be(addr)); return 0xffff; } -unsigned int ioread32(void __iomem *addr) +unsigned int ioread32(const void __iomem *addr) { IO_COND(addr, return inl(port), return readl(addr)); return 0xffffffff; } -unsigned int ioread32be(void __iomem *addr) +unsigned int ioread32be(const void __iomem *addr) { IO_COND(addr, return pio_read32be(port), return mmio_read32be(addr)); return 0xffffffff; @@ -142,26 +142,26 @@ static u64 pio_read64be_hi_lo(unsigned long port) return lo | (hi << 32); } -u64 ioread64_lo_hi(void __iomem *addr) +u64 ioread64_lo_hi(const void __iomem *addr) { IO_COND(addr, return pio_read64_lo_hi(port), return readq(addr)); return 0xffffffffffffffffULL; } -u64 ioread64_hi_lo(void __iomem *addr) +u64 ioread64_hi_lo(const void __iomem *addr) { IO_COND(addr, return pio_read64_hi_lo(port), return readq(addr)); return 0xffffffffffffffffULL; } -u64 ioread64be_lo_hi(void __iomem *addr) +u64 ioread64be_lo_hi(const void __iomem *addr) { IO_COND(addr, return pio_read64be_lo_hi(port), return mmio_read64be(addr)); return 0xffffffffffffffffULL; } -u64 ioread64be_hi_lo(void __iomem *addr) +u64 ioread64be_hi_lo(const void __iomem *addr) { IO_COND(addr, return pio_read64be_hi_lo(port), return mmio_read64be(addr)); @@ -275,7 +275,7 @@ EXPORT_SYMBOL(iowrite64be_hi_lo); * order" (we also don't have IO barriers). */ #ifndef mmio_insb -static inline void mmio_insb(void __iomem *addr, u8 *dst, int count) +static inline void mmio_insb(const void __iomem *addr, u8 *dst, int count) { while (--count >= 0) { u8 data = __raw_readb(addr); @@ -283,7 +283,7 @@ static inline void mmio_insb(void __iomem *addr, u8 *dst, int count) dst++; } } -static inline void mmio_insw(void __iomem *addr, u16 *dst, int count) +static inline void mmio_insw(const void __iomem *addr, u16 *dst, int count) { while (--count >= 0) { u16 data = __raw_readw(addr); @@ -291,7 +291,7 @@ static inline void mmio_insw(void __iomem *addr, u16 *dst, int count) dst++; } } -static inline void mmio_insl(void __iomem *addr, u32 *dst, int count) +static inline void mmio_insl(const void __iomem *addr, u32 *dst, int count) { while (--count >= 0) { u32 data = __raw_readl(addr); @@ -325,15 +325,15 @@ static inline void mmio_outsl(void __iomem *addr, const u32 *src, int count) } #endif -void ioread8_rep(void __iomem *addr, void *dst, unsigned long count) +void ioread8_rep(const void __iomem *addr, void *dst, unsigned long count) { IO_COND(addr, insb(port,dst,count), mmio_insb(addr, dst, count)); } -void ioread16_rep(void __iomem *addr, void *dst, unsigned long count) +void ioread16_rep(const void __iomem *addr, void *dst, unsigned long count) { IO_COND(addr, insw(port,dst,count), mmio_insw(addr, dst, count)); } -void ioread32_rep(void __iomem *addr, void *dst, unsigned long count) +void ioread32_rep(const void __iomem *addr, void *dst, unsigned long count) { IO_COND(addr, insl(port,dst,count), mmio_insl(addr, dst, count)); } diff --git a/lib/ioremap.c b/lib/ioremap.c deleted file mode 100644 index 3f0e18543de8..000000000000 --- a/lib/ioremap.c +++ /dev/null @@ -1,272 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Re-map IO memory to kernel address space so that we can access it. - * This is needed for high PCI addresses that aren't mapped in the - * 640k-1MB IO memory area on PC's - * - * (C) Copyright 1995 1996 Linus Torvalds - */ -#include <linux/vmalloc.h> -#include <linux/mm.h> -#include <linux/sched.h> -#include <linux/io.h> -#include <linux/export.h> -#include <asm/cacheflush.h> -#include <asm/pgtable.h> - -#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP -static int __read_mostly ioremap_p4d_capable; -static int __read_mostly ioremap_pud_capable; -static int __read_mostly ioremap_pmd_capable; -static int __read_mostly ioremap_huge_disabled; - -static int __init set_nohugeiomap(char *str) -{ - ioremap_huge_disabled = 1; - return 0; -} -early_param("nohugeiomap", set_nohugeiomap); - -void __init ioremap_huge_init(void) -{ - if (!ioremap_huge_disabled) { - if (arch_ioremap_p4d_supported()) - ioremap_p4d_capable = 1; - if (arch_ioremap_pud_supported()) - ioremap_pud_capable = 1; - if (arch_ioremap_pmd_supported()) - ioremap_pmd_capable = 1; - } -} - -static inline int ioremap_p4d_enabled(void) -{ - return ioremap_p4d_capable; -} - -static inline int ioremap_pud_enabled(void) -{ - return ioremap_pud_capable; -} - -static inline int ioremap_pmd_enabled(void) -{ - return ioremap_pmd_capable; -} - -#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ -static inline int ioremap_p4d_enabled(void) { return 0; } -static inline int ioremap_pud_enabled(void) { return 0; } -static inline int ioremap_pmd_enabled(void) { return 0; } -#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ - -static int ioremap_pte_range(pmd_t *pmd, unsigned long addr, - unsigned long end, phys_addr_t phys_addr, pgprot_t prot) -{ - pte_t *pte; - u64 pfn; - - pfn = phys_addr >> PAGE_SHIFT; - pte = pte_alloc_kernel(pmd, addr); - if (!pte) - return -ENOMEM; - do { - BUG_ON(!pte_none(*pte)); - set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); - pfn++; - } while (pte++, addr += PAGE_SIZE, addr != end); - return 0; -} - -static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr, - unsigned long end, phys_addr_t phys_addr, - pgprot_t prot) -{ - if (!ioremap_pmd_enabled()) - return 0; - - if ((end - addr) != PMD_SIZE) - return 0; - - if (!IS_ALIGNED(addr, PMD_SIZE)) - return 0; - - if (!IS_ALIGNED(phys_addr, PMD_SIZE)) - return 0; - - if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) - return 0; - - return pmd_set_huge(pmd, phys_addr, prot); -} - -static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr, - unsigned long end, phys_addr_t phys_addr, pgprot_t prot) -{ - pmd_t *pmd; - unsigned long next; - - pmd = pmd_alloc(&init_mm, pud, addr); - if (!pmd) - return -ENOMEM; - do { - next = pmd_addr_end(addr, end); - - if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr, prot)) - continue; - - if (ioremap_pte_range(pmd, addr, next, phys_addr, prot)) - return -ENOMEM; - } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); - return 0; -} - -static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr, - unsigned long end, phys_addr_t phys_addr, - pgprot_t prot) -{ - if (!ioremap_pud_enabled()) - return 0; - - if ((end - addr) != PUD_SIZE) - return 0; - - if (!IS_ALIGNED(addr, PUD_SIZE)) - return 0; - - if (!IS_ALIGNED(phys_addr, PUD_SIZE)) - return 0; - - if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) - return 0; - - return pud_set_huge(pud, phys_addr, prot); -} - -static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, - unsigned long end, phys_addr_t phys_addr, pgprot_t prot) -{ - pud_t *pud; - unsigned long next; - - pud = pud_alloc(&init_mm, p4d, addr); - if (!pud) - return -ENOMEM; - do { - next = pud_addr_end(addr, end); - - if (ioremap_try_huge_pud(pud, addr, next, phys_addr, prot)) - continue; - - if (ioremap_pmd_range(pud, addr, next, phys_addr, prot)) - return -ENOMEM; - } while (pud++, phys_addr += (next - addr), addr = next, addr != end); - return 0; -} - -static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr, - unsigned long end, phys_addr_t phys_addr, - pgprot_t prot) -{ - if (!ioremap_p4d_enabled()) - return 0; - - if ((end - addr) != P4D_SIZE) - return 0; - - if (!IS_ALIGNED(addr, P4D_SIZE)) - return 0; - - if (!IS_ALIGNED(phys_addr, P4D_SIZE)) - return 0; - - if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) - return 0; - - return p4d_set_huge(p4d, phys_addr, prot); -} - -static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, - unsigned long end, phys_addr_t phys_addr, pgprot_t prot) -{ - p4d_t *p4d; - unsigned long next; - - p4d = p4d_alloc(&init_mm, pgd, addr); - if (!p4d) - return -ENOMEM; - do { - next = p4d_addr_end(addr, end); - - if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) - continue; - - if (ioremap_pud_range(p4d, addr, next, phys_addr, prot)) - return -ENOMEM; - } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); - return 0; -} - -int ioremap_page_range(unsigned long addr, - unsigned long end, phys_addr_t phys_addr, pgprot_t prot) -{ - pgd_t *pgd; - unsigned long start; - unsigned long next; - int err; - - might_sleep(); - BUG_ON(addr >= end); - - start = addr; - pgd = pgd_offset_k(addr); - do { - next = pgd_addr_end(addr, end); - err = ioremap_p4d_range(pgd, addr, next, phys_addr, prot); - if (err) - break; - } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); - - flush_cache_vmap(start, end); - - return err; -} - -#ifdef CONFIG_GENERIC_IOREMAP -void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot) -{ - unsigned long offset, vaddr; - phys_addr_t last_addr; - struct vm_struct *area; - - /* Disallow wrap-around or zero size */ - last_addr = addr + size - 1; - if (!size || last_addr < addr) - return NULL; - - /* Page-align mappings */ - offset = addr & (~PAGE_MASK); - addr -= offset; - size = PAGE_ALIGN(size + offset); - - area = get_vm_area_caller(size, VM_IOREMAP, - __builtin_return_address(0)); - if (!area) - return NULL; - vaddr = (unsigned long)area->addr; - - if (ioremap_page_range(vaddr, vaddr + size, addr, __pgprot(prot))) { - free_vm_area(area); - return NULL; - } - - return (void __iomem *)(vaddr + offset); -} -EXPORT_SYMBOL(ioremap_prot); - -void iounmap(volatile void __iomem *addr) -{ - vunmap((void *)((unsigned long)addr & PAGE_MASK)); -} -EXPORT_SYMBOL(iounmap); -#endif /* CONFIG_GENERIC_IOREMAP */ diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 51595bf3af85..66a740e6e153 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1,144 +1,159 @@ // SPDX-License-Identifier: GPL-2.0-only +#include <crypto/hash.h> #include <linux/export.h> #include <linux/bvec.h> +#include <linux/fault-inject-usercopy.h> #include <linux/uio.h> #include <linux/pagemap.h> +#include <linux/highmem.h> #include <linux/slab.h> #include <linux/vmalloc.h> #include <linux/splice.h> +#include <linux/compat.h> #include <net/checksum.h> #include <linux/scatterlist.h> +#include <linux/instrumented.h> #define PIPE_PARANOIA /* for now */ -#define iterate_iovec(i, n, __v, __p, skip, STEP) { \ - size_t left; \ - size_t wanted = n; \ - __p = i->iov; \ - __v.iov_len = min(n, __p->iov_len - skip); \ - if (likely(__v.iov_len)) { \ - __v.iov_base = __p->iov_base + skip; \ - left = (STEP); \ - __v.iov_len -= left; \ - skip += __v.iov_len; \ - n -= __v.iov_len; \ - } else { \ - left = 0; \ - } \ - while (unlikely(!left && n)) { \ - __p++; \ - __v.iov_len = min(n, __p->iov_len); \ - if (unlikely(!__v.iov_len)) \ - continue; \ - __v.iov_base = __p->iov_base; \ - left = (STEP); \ - __v.iov_len -= left; \ - skip = __v.iov_len; \ - n -= __v.iov_len; \ - } \ - n = wanted - n; \ -} - -#define iterate_kvec(i, n, __v, __p, skip, STEP) { \ - size_t wanted = n; \ - __p = i->kvec; \ - __v.iov_len = min(n, __p->iov_len - skip); \ - if (likely(__v.iov_len)) { \ - __v.iov_base = __p->iov_base + skip; \ - (void)(STEP); \ - skip += __v.iov_len; \ - n -= __v.iov_len; \ - } \ - while (unlikely(n)) { \ - __p++; \ - __v.iov_len = min(n, __p->iov_len); \ - if (unlikely(!__v.iov_len)) \ - continue; \ - __v.iov_base = __p->iov_base; \ - (void)(STEP); \ - skip = __v.iov_len; \ - n -= __v.iov_len; \ - } \ - n = wanted; \ -} - -#define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ - struct bvec_iter __start; \ - __start.bi_size = n; \ - __start.bi_bvec_done = skip; \ - __start.bi_idx = 0; \ - for_each_bvec(__v, i->bvec, __bi, __start) { \ - if (!__v.bv_len) \ - continue; \ - (void)(STEP); \ - } \ -} - -#define iterate_all_kinds(i, n, v, I, B, K) { \ - if (likely(n)) { \ - size_t skip = i->iov_offset; \ - if (unlikely(i->type & ITER_BVEC)) { \ - struct bio_vec v; \ - struct bvec_iter __bi; \ - iterate_bvec(i, n, v, __bi, skip, (B)) \ - } else if (unlikely(i->type & ITER_KVEC)) { \ - const struct kvec *kvec; \ - struct kvec v; \ - iterate_kvec(i, n, v, kvec, skip, (K)) \ - } else if (unlikely(i->type & ITER_DISCARD)) { \ - } else { \ - const struct iovec *iov; \ - struct iovec v; \ - iterate_iovec(i, n, v, iov, skip, (I)) \ +/* covers iovec and kvec alike */ +#define iterate_iovec(i, n, base, len, off, __p, STEP) { \ + size_t off = 0; \ + size_t skip = i->iov_offset; \ + do { \ + len = min(n, __p->iov_len - skip); \ + if (likely(len)) { \ + base = __p->iov_base + skip; \ + len -= (STEP); \ + off += len; \ + skip += len; \ + n -= len; \ + if (skip < __p->iov_len) \ + break; \ } \ + __p++; \ + skip = 0; \ + } while (n); \ + i->iov_offset = skip; \ + n = off; \ +} + +#define iterate_bvec(i, n, base, len, off, p, STEP) { \ + size_t off = 0; \ + unsigned skip = i->iov_offset; \ + while (n) { \ + unsigned offset = p->bv_offset + skip; \ + unsigned left; \ + void *kaddr = kmap_local_page(p->bv_page + \ + offset / PAGE_SIZE); \ + base = kaddr + offset % PAGE_SIZE; \ + len = min(min(n, (size_t)(p->bv_len - skip)), \ + (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ + left = (STEP); \ + kunmap_local(kaddr); \ + len -= left; \ + off += len; \ + skip += len; \ + if (skip == p->bv_len) { \ + skip = 0; \ + p++; \ + } \ + n -= len; \ + if (left) \ + break; \ } \ + i->iov_offset = skip; \ + n = off; \ +} + +#define iterate_xarray(i, n, base, len, __off, STEP) { \ + __label__ __out; \ + size_t __off = 0; \ + struct page *head = NULL; \ + loff_t start = i->xarray_start + i->iov_offset; \ + unsigned offset = start % PAGE_SIZE; \ + pgoff_t index = start / PAGE_SIZE; \ + int j; \ + \ + XA_STATE(xas, i->xarray, index); \ + \ + rcu_read_lock(); \ + xas_for_each(&xas, head, ULONG_MAX) { \ + unsigned left; \ + if (xas_retry(&xas, head)) \ + continue; \ + if (WARN_ON(xa_is_value(head))) \ + break; \ + if (WARN_ON(PageHuge(head))) \ + break; \ + for (j = (head->index < index) ? index - head->index : 0; \ + j < thp_nr_pages(head); j++) { \ + void *kaddr = kmap_local_page(head + j); \ + base = kaddr + offset; \ + len = PAGE_SIZE - offset; \ + len = min(n, len); \ + left = (STEP); \ + kunmap_local(kaddr); \ + len -= left; \ + __off += len; \ + n -= len; \ + if (left || n == 0) \ + goto __out; \ + offset = 0; \ + } \ + } \ +__out: \ + rcu_read_unlock(); \ + i->iov_offset += __off; \ + n = __off; \ } -#define iterate_and_advance(i, n, v, I, B, K) { \ +#define __iterate_and_advance(i, n, base, len, off, I, K) { \ if (unlikely(i->count < n)) \ n = i->count; \ - if (i->count) { \ - size_t skip = i->iov_offset; \ - if (unlikely(i->type & ITER_BVEC)) { \ + if (likely(n)) { \ + if (likely(iter_is_iovec(i))) { \ + const struct iovec *iov = i->iov; \ + void __user *base; \ + size_t len; \ + iterate_iovec(i, n, base, len, off, \ + iov, (I)) \ + i->nr_segs -= iov - i->iov; \ + i->iov = iov; \ + } else if (iov_iter_is_bvec(i)) { \ const struct bio_vec *bvec = i->bvec; \ - struct bio_vec v; \ - struct bvec_iter __bi; \ - iterate_bvec(i, n, v, __bi, skip, (B)) \ - i->bvec = __bvec_iter_bvec(i->bvec, __bi); \ - i->nr_segs -= i->bvec - bvec; \ - skip = __bi.bi_bvec_done; \ - } else if (unlikely(i->type & ITER_KVEC)) { \ - const struct kvec *kvec; \ - struct kvec v; \ - iterate_kvec(i, n, v, kvec, skip, (K)) \ - if (skip == kvec->iov_len) { \ - kvec++; \ - skip = 0; \ - } \ + void *base; \ + size_t len; \ + iterate_bvec(i, n, base, len, off, \ + bvec, (K)) \ + i->nr_segs -= bvec - i->bvec; \ + i->bvec = bvec; \ + } else if (iov_iter_is_kvec(i)) { \ + const struct kvec *kvec = i->kvec; \ + void *base; \ + size_t len; \ + iterate_iovec(i, n, base, len, off, \ + kvec, (K)) \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ - } else if (unlikely(i->type & ITER_DISCARD)) { \ - skip += n; \ - } else { \ - const struct iovec *iov; \ - struct iovec v; \ - iterate_iovec(i, n, v, iov, skip, (I)) \ - if (skip == iov->iov_len) { \ - iov++; \ - skip = 0; \ - } \ - i->nr_segs -= iov - i->iov; \ - i->iov = iov; \ + } else if (iov_iter_is_xarray(i)) { \ + void *base; \ + size_t len; \ + iterate_xarray(i, n, base, len, off, \ + (K)) \ } \ i->count -= n; \ - i->iov_offset = skip; \ } \ } +#define iterate_and_advance(i, n, base, len, off, I, K) \ + __iterate_and_advance(i, n, base, len, off, I, ((void)(K),0)) static int copyout(void __user *to, const void *from, size_t n) { + if (should_fail_usercopy()) + return n; if (access_ok(to, n)) { - kasan_check_read(from, n); + instrument_copy_to_user(to, from, n); n = raw_copy_to_user(to, from, n); } return n; @@ -146,8 +161,10 @@ static int copyout(void __user *to, const void *from, size_t n) static int copyin(void *to, const void __user *from, size_t n) { + if (should_fail_usercopy()) + return n; if (access_ok(from, n)) { - kasan_check_write(to, n); + instrument_copy_from_user(to, from, n); n = raw_copy_from_user(to, from, n); } return n; @@ -174,7 +191,7 @@ static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t b buf = iov->iov_base + skip; copy = min(bytes, iov->iov_len - skip); - if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) { + if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_writeable(buf, copy)) { kaddr = kmap_atomic(page); from = kaddr + offset; @@ -258,7 +275,7 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t buf = iov->iov_base + skip; copy = min(bytes, iov->iov_len - skip); - if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) { + if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_readable(buf, copy)) { kaddr = kmap_atomic(page); to = kaddr + offset; @@ -413,72 +430,99 @@ out: } /* + * fault_in_iov_iter_readable - fault in iov iterator for reading + * @i: iterator + * @size: maximum length + * * Fault in one or more iovecs of the given iov_iter, to a maximum length of - * bytes. For each iovec, fault in each page that constitutes the iovec. + * @size. For each iovec, fault in each page that constitutes the iovec. + * + * Returns the number of bytes not faulted in (like copy_to_user() and + * copy_from_user()). * - * Return 0 on success, or non-zero if the memory could not be accessed (i.e. - * because it is an invalid address). + * Always returns 0 for non-userspace iterators. */ -int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) +size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) { - size_t skip = i->iov_offset; - const struct iovec *iov; - int err; - struct iovec v; - - if (!(i->type & (ITER_BVEC|ITER_KVEC))) { - iterate_iovec(i, bytes, v, iov, skip, ({ - err = fault_in_pages_readable(v.iov_base, v.iov_len); - if (unlikely(err)) - return err; - 0;})) + if (iter_is_iovec(i)) { + size_t count = min(size, iov_iter_count(i)); + const struct iovec *p; + size_t skip; + + size -= count; + for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { + size_t len = min(count, p->iov_len - skip); + size_t ret; + + if (unlikely(!len)) + continue; + ret = fault_in_readable(p->iov_base + skip, len); + count -= len - ret; + if (ret) + break; + } + return count + size; } return 0; } -EXPORT_SYMBOL(iov_iter_fault_in_readable); +EXPORT_SYMBOL(fault_in_iov_iter_readable); + +/* + * fault_in_iov_iter_writeable - fault in iov iterator for writing + * @i: iterator + * @size: maximum length + * + * Faults in the iterator using get_user_pages(), i.e., without triggering + * hardware page faults. This is primarily useful when we already know that + * some or all of the pages in @i aren't in memory. + * + * Returns the number of bytes not faulted in, like copy_to_user() and + * copy_from_user(). + * + * Always returns 0 for non-user-space iterators. + */ +size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) +{ + if (iter_is_iovec(i)) { + size_t count = min(size, iov_iter_count(i)); + const struct iovec *p; + size_t skip; + + size -= count; + for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { + size_t len = min(count, p->iov_len - skip); + size_t ret; + + if (unlikely(!len)) + continue; + ret = fault_in_safe_writeable(p->iov_base + skip, len); + count -= len - ret; + if (ret) + break; + } + return count + size; + } + return 0; +} +EXPORT_SYMBOL(fault_in_iov_iter_writeable); void iov_iter_init(struct iov_iter *i, unsigned int direction, const struct iovec *iov, unsigned long nr_segs, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - direction &= READ | WRITE; - - /* It will get better. Eventually... */ - if (uaccess_kernel()) { - i->type = ITER_KVEC | direction; - i->kvec = (struct kvec *)iov; - } else { - i->type = ITER_IOVEC | direction; - i->iov = iov; - } - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter) { + .iter_type = ITER_IOVEC, + .nofault = false, + .data_source = direction, + .iov = iov, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_init); -static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) -{ - char *from = kmap_atomic(page); - memcpy(to, from + offset, len); - kunmap_atomic(from); -} - -static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len) -{ - char *to = kmap_atomic(page); - memcpy(to + offset, from, len); - kunmap_atomic(to); -} - -static void memzero_page(struct page *page, size_t offset, size_t len) -{ - char *addr = kmap_atomic(page); - memset(addr + offset, 0, len); - kunmap_atomic(addr); -} - static inline bool allocated(struct pipe_buffer *buf) { return buf->ops == &default_pipe_buf_ops; @@ -579,86 +623,67 @@ static size_t copy_pipe_to_iter(const void *addr, size_t bytes, static __wsum csum_and_memcpy(void *to, const void *from, size_t len, __wsum sum, size_t off) { - __wsum next = csum_partial_copy_nocheck(from, to, len, 0); + __wsum next = csum_partial_copy_nocheck(from, to, len); return csum_block_add(sum, next, off); } static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, - __wsum *csum, struct iov_iter *i) + struct iov_iter *i, __wsum *sump) { struct pipe_inode_info *pipe = i->pipe; unsigned int p_mask = pipe->ring_size - 1; - unsigned int i_head; - size_t n, r; + __wsum sum = *sump; size_t off = 0; - __wsum sum = *csum; + unsigned int i_head; + size_t r; if (!sanity(i)) return 0; - bytes = n = push_pipe(i, bytes, &i_head, &r); - if (unlikely(!n)) - return 0; - do { - size_t chunk = min_t(size_t, n, PAGE_SIZE - r); - char *p = kmap_atomic(pipe->bufs[i_head & p_mask].page); - sum = csum_and_memcpy(p + r, addr, chunk, sum, off); - kunmap_atomic(p); + bytes = push_pipe(i, bytes, &i_head, &r); + while (bytes) { + size_t chunk = min_t(size_t, bytes, PAGE_SIZE - r); + char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page); + sum = csum_and_memcpy(p + r, addr + off, chunk, sum, off); + kunmap_local(p); i->head = i_head; i->iov_offset = r + chunk; - n -= chunk; + bytes -= chunk; off += chunk; - addr += chunk; r = 0; i_head++; - } while (n); - i->count -= bytes; - *csum = sum; - return bytes; + } + *sump = sum; + i->count -= off; + return off; } size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { - const char *from = addr; if (unlikely(iov_iter_is_pipe(i))) return copy_pipe_to_iter(addr, bytes, i); if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, - copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - memcpy_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len), - memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) + iterate_and_advance(i, bytes, base, len, off, + copyout(base, addr + off, len), + memcpy(base, addr + off, len) ) return bytes; } EXPORT_SYMBOL(_copy_to_iter); -#ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE -static int copyout_mcsafe(void __user *to, const void *from, size_t n) +#ifdef CONFIG_ARCH_HAS_COPY_MC +static int copyout_mc(void __user *to, const void *from, size_t n) { if (access_ok(to, n)) { - kasan_check_read(from, n); - n = copy_to_user_mcsafe((__force void *) to, from, n); + instrument_copy_to_user(to, from, n); + n = copy_mc_to_user((__force void *) to, from, n); } return n; } -static unsigned long memcpy_mcsafe_to_page(struct page *page, size_t offset, - const char *from, size_t len) -{ - unsigned long ret; - char *to; - - to = kmap_atomic(page); - ret = memcpy_mcsafe(to + offset, from, len); - kunmap_atomic(to); - - return ret; -} - -static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes, +static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { struct pipe_inode_info *pipe = i->pipe; @@ -669,42 +694,39 @@ static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes, if (!sanity(i)) return 0; - bytes = n = push_pipe(i, bytes, &i_head, &off); - if (unlikely(!n)) - return 0; - do { + n = push_pipe(i, bytes, &i_head, &off); + while (n) { size_t chunk = min_t(size_t, n, PAGE_SIZE - off); + char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page); unsigned long rem; - - rem = memcpy_mcsafe_to_page(pipe->bufs[i_head & p_mask].page, - off, addr, chunk); + rem = copy_mc_to_kernel(p + off, addr + xfer, chunk); + chunk -= rem; + kunmap_local(p); i->head = i_head; - i->iov_offset = off + chunk - rem; - xfer += chunk - rem; + i->iov_offset = off + chunk; + xfer += chunk; if (rem) break; n -= chunk; - addr += chunk; off = 0; i_head++; - } while (n); + } i->count -= xfer; return xfer; } /** - * _copy_to_iter_mcsafe - copy to user with source-read error exception handling + * _copy_mc_to_iter - copy to iter with source memory error exception handling * @addr: source kernel address * @bytes: total transfer length - * @iter: destination iterator + * @i: destination iterator * - * The pmem driver arranges for filesystem-dax to use this facility via - * dax_copy_to_iter() for protecting read/write to persistent memory. - * Unless / until an architecture can guarantee identical performance - * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a - * performance regression to switch more users to the mcsafe version. + * The pmem driver deploys this for the dax operation + * (dax_copy_to_iter()) for dax reads (bypass page-cache and the + * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes + * successfully copied. * - * Otherwise, the main differences between this and typical _copy_to_iter(). + * The main differences between this and typical _copy_to_iter(). * * * Typical tail/residue handling after a fault retries the copy * byte-by-byte until the fault happens again. Re-triggering machine @@ -716,104 +738,50 @@ static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes, * Compare to copy_to_iter() where only ITER_IOVEC attempts might return * a short copy. * - * See MCSAFE_TEST for self-test. + * Return: number of bytes copied (may be %0) */ -size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i) +size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { - const char *from = addr; - unsigned long rem, curr_addr, s_addr = (unsigned long) addr; - if (unlikely(iov_iter_is_pipe(i))) - return copy_pipe_to_iter_mcsafe(addr, bytes, i); + return copy_mc_pipe_to_iter(addr, bytes, i); if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, - copyout_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - ({ - rem = memcpy_mcsafe_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len); - if (rem) { - curr_addr = (unsigned long) from; - bytes = curr_addr - s_addr - rem; - return bytes; - } - }), - ({ - rem = memcpy_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, - v.iov_len); - if (rem) { - curr_addr = (unsigned long) from; - bytes = curr_addr - s_addr - rem; - return bytes; - } - }) + __iterate_and_advance(i, bytes, base, len, off, + copyout_mc(base, addr + off, len), + copy_mc_to_kernel(base, addr + off, len) ) return bytes; } -EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe); -#endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */ +EXPORT_SYMBOL_GPL(_copy_mc_to_iter); +#endif /* CONFIG_ARCH_HAS_COPY_MC */ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; if (unlikely(iov_iter_is_pipe(i))) { WARN_ON(1); return 0; } if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, - copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) + iterate_and_advance(i, bytes, base, len, off, + copyin(addr + off, base, len), + memcpy(addr + off, base, len) ) return bytes; } EXPORT_SYMBOL(_copy_from_iter); -bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) -{ - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - - if (iter_is_iovec(i)) - might_fault(); - iterate_all_kinds(i, bytes, v, ({ - if (copyin((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len)) - return false; - 0;}), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) - ) - - iov_iter_advance(i, bytes); - return true; -} -EXPORT_SYMBOL(_copy_from_iter_full); - size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; if (unlikely(iov_iter_is_pipe(i))) { WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, - __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) + iterate_and_advance(i, bytes, base, len, off, + __copy_from_user_inatomic_nocache(addr + off, base, len), + memcpy(addr + off, base, len) ) return bytes; @@ -825,7 +793,7 @@ EXPORT_SYMBOL(_copy_from_iter_nocache); * _copy_from_iter_flushcache - write destination through cpu cache * @addr: destination kernel address * @bytes: total transfer length - * @iter: source iterator + * @i: source iterator * * The pmem driver arranges for filesystem-dax to use this facility via * dax_copy_from_iter() for ensuring that writes to persistent memory @@ -834,21 +802,18 @@ EXPORT_SYMBOL(_copy_from_iter_nocache); * all iterator types. The _copy_from_iter_nocache() only attempts to * bypass the cache for the ITER_IOVEC case, and on some archs may use * instructions that strand dirty-data in the cache. + * + * Return: number of bytes copied (may be %0) */ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; if (unlikely(iov_iter_is_pipe(i))) { WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, - __copy_from_user_flushcache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len), - memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base, - v.iov_len) + iterate_and_advance(i, bytes, base, len, off, + __copy_from_user_flushcache(addr + off, base, len), + memcpy_flushcache(addr + off, base, len) ) return bytes; @@ -856,30 +821,6 @@ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache); #endif -bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) -{ - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - iterate_all_kinds(i, bytes, v, ({ - if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len)) - return false; - 0;}), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) - ) - - iov_iter_advance(i, bytes); - return true; -} -EXPORT_SYMBOL(_copy_from_iter_full_nocache); - static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) { struct page *head; @@ -904,22 +845,51 @@ static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) return false; } +static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes, + struct iov_iter *i) +{ + if (likely(iter_is_iovec(i))) + return copy_page_to_iter_iovec(page, offset, bytes, i); + if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) { + void *kaddr = kmap_local_page(page); + size_t wanted = _copy_to_iter(kaddr + offset, bytes, i); + kunmap_local(kaddr); + return wanted; + } + if (iov_iter_is_pipe(i)) + return copy_page_to_iter_pipe(page, offset, bytes, i); + if (unlikely(iov_iter_is_discard(i))) { + if (unlikely(i->count < bytes)) + bytes = i->count; + i->count -= bytes; + return bytes; + } + WARN_ON(1); + return 0; +} + size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, struct iov_iter *i) { + size_t res = 0; if (unlikely(!page_copy_sane(page, offset, bytes))) return 0; - if (i->type & (ITER_BVEC|ITER_KVEC)) { - void *kaddr = kmap_atomic(page); - size_t wanted = copy_to_iter(kaddr + offset, bytes, i); - kunmap_atomic(kaddr); - return wanted; - } else if (unlikely(iov_iter_is_discard(i))) - return bytes; - else if (likely(!iov_iter_is_pipe(i))) - return copy_page_to_iter_iovec(page, offset, bytes, i); - else - return copy_page_to_iter_pipe(page, offset, bytes, i); + page += offset / PAGE_SIZE; // first subpage + offset %= PAGE_SIZE; + while (1) { + size_t n = __copy_page_to_iter(page, offset, + min(bytes, (size_t)PAGE_SIZE - offset), i); + res += n; + bytes -= n; + if (!bytes || !n) + break; + offset += n; + if (offset == PAGE_SIZE) { + page++; + offset = 0; + } + } + return res; } EXPORT_SYMBOL(copy_page_to_iter); @@ -928,17 +898,16 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, { if (unlikely(!page_copy_sane(page, offset, bytes))) return 0; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); - return 0; - } - if (i->type & (ITER_BVEC|ITER_KVEC)) { - void *kaddr = kmap_atomic(page); + if (likely(iter_is_iovec(i))) + return copy_page_from_iter_iovec(page, offset, bytes, i); + if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) { + void *kaddr = kmap_local_page(page); size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); - kunmap_atomic(kaddr); + kunmap_local(kaddr); return wanted; - } else - return copy_page_from_iter_iovec(page, offset, bytes, i); + } + WARN_ON(1); + return 0; } EXPORT_SYMBOL(copy_page_from_iter); @@ -958,7 +927,9 @@ static size_t pipe_zero(size_t bytes, struct iov_iter *i) do { size_t chunk = min_t(size_t, n, PAGE_SIZE - off); - memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk); + char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page); + memset(p + off, 0, chunk); + kunmap_local(p); i->head = i_head; i->iov_offset = off + chunk; n -= chunk; @@ -973,18 +944,17 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i) { if (unlikely(iov_iter_is_pipe(i))) return pipe_zero(bytes, i); - iterate_and_advance(i, bytes, v, - clear_user(v.iov_base, v.iov_len), - memzero_page(v.bv_page, v.bv_offset, v.bv_len), - memset(v.iov_base, 0, v.iov_len) + iterate_and_advance(i, bytes, base, len, count, + clear_user(base, len), + memset(base, 0, len) ) return bytes; } EXPORT_SYMBOL(iov_iter_zero); -size_t iov_iter_copy_from_user_atomic(struct page *page, - struct iov_iter *i, unsigned long offset, size_t bytes) +size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t bytes, + struct iov_iter *i) { char *kaddr = kmap_atomic(page), *p = kaddr + offset; if (unlikely(!page_copy_sane(page, offset, bytes))) { @@ -996,16 +966,14 @@ size_t iov_iter_copy_from_user_atomic(struct page *page, WARN_ON(1); return 0; } - iterate_all_kinds(i, bytes, v, - copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) + iterate_and_advance(i, bytes, base, len, off, + copyin(p + off, base, len), + memcpy(p + off, base, len) ) kunmap_atomic(kaddr); return bytes; } -EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); +EXPORT_SYMBOL(copy_page_from_iter_atomic); static inline void pipe_truncate(struct iov_iter *i) { @@ -1036,8 +1004,6 @@ static inline void pipe_truncate(struct iov_iter *i) static void pipe_advance(struct iov_iter *i, size_t size) { struct pipe_inode_info *pipe = i->pipe; - if (unlikely(i->count < size)) - size = i->count; if (size) { struct pipe_buffer *buf; unsigned int p_mask = pipe->ring_size - 1; @@ -1061,17 +1027,57 @@ static void pipe_advance(struct iov_iter *i, size_t size) pipe_truncate(i); } -void iov_iter_advance(struct iov_iter *i, size_t size) +static void iov_iter_bvec_advance(struct iov_iter *i, size_t size) { - if (unlikely(iov_iter_is_pipe(i))) { - pipe_advance(i, size); + struct bvec_iter bi; + + bi.bi_size = i->count; + bi.bi_bvec_done = i->iov_offset; + bi.bi_idx = 0; + bvec_iter_advance(i->bvec, &bi, size); + + i->bvec += bi.bi_idx; + i->nr_segs -= bi.bi_idx; + i->count = bi.bi_size; + i->iov_offset = bi.bi_bvec_done; +} + +static void iov_iter_iovec_advance(struct iov_iter *i, size_t size) +{ + const struct iovec *iov, *end; + + if (!i->count) return; + i->count -= size; + + size += i->iov_offset; // from beginning of current segment + for (iov = i->iov, end = iov + i->nr_segs; iov < end; iov++) { + if (likely(size < iov->iov_len)) + break; + size -= iov->iov_len; } - if (unlikely(iov_iter_is_discard(i))) { + i->iov_offset = size; + i->nr_segs -= iov - i->iov; + i->iov = iov; +} + +void iov_iter_advance(struct iov_iter *i, size_t size) +{ + if (unlikely(i->count < size)) + size = i->count; + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) { + /* iovec and kvec have identical layouts */ + iov_iter_iovec_advance(i, size); + } else if (iov_iter_is_bvec(i)) { + iov_iter_bvec_advance(i, size); + } else if (iov_iter_is_pipe(i)) { + pipe_advance(i, size); + } else if (unlikely(iov_iter_is_xarray(i))) { + i->iov_offset += size; + i->count -= size; + } else if (iov_iter_is_discard(i)) { i->count -= size; - return; } - iterate_and_advance(i, size, v, 0, 0, 0) } EXPORT_SYMBOL(iov_iter_advance); @@ -1115,7 +1121,12 @@ void iov_iter_revert(struct iov_iter *i, size_t unroll) return; } unroll -= i->iov_offset; - if (iov_iter_is_bvec(i)) { + if (iov_iter_is_xarray(i)) { + BUG(); /* We should never go beyond the start of the specified + * range since we might then be straying into pages that + * aren't pinned. + */ + } else if (iov_iter_is_bvec(i)) { const struct bio_vec *bvec = i->bvec; while (1) { size_t n = (--bvec)->bv_len; @@ -1148,16 +1159,13 @@ EXPORT_SYMBOL(iov_iter_revert); */ size_t iov_iter_single_seg_count(const struct iov_iter *i) { - if (unlikely(iov_iter_is_pipe(i))) - return i->count; // it is a silly place, anyway - if (i->nr_segs == 1) - return i->count; - if (unlikely(iov_iter_is_discard(i))) - return i->count; - else if (iov_iter_is_bvec(i)) - return min(i->count, i->bvec->bv_len - i->iov_offset); - else - return min(i->count, i->iov->iov_len - i->iov_offset); + if (i->nr_segs > 1) { + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return min(i->count, i->iov->iov_len - i->iov_offset); + if (iov_iter_is_bvec(i)) + return min(i->count, i->bvec->bv_len - i->iov_offset); + } + return i->count; } EXPORT_SYMBOL(iov_iter_single_seg_count); @@ -1166,11 +1174,14 @@ void iov_iter_kvec(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - i->type = ITER_KVEC | (direction & (READ | WRITE)); - i->kvec = kvec; - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter){ + .iter_type = ITER_KVEC, + .data_source = direction, + .kvec = kvec, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_kvec); @@ -1179,11 +1190,14 @@ void iov_iter_bvec(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - i->type = ITER_BVEC | (direction & (READ | WRITE)); - i->bvec = bvec; - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter){ + .iter_type = ITER_BVEC, + .data_source = direction, + .bvec = bvec, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_bvec); @@ -1193,16 +1207,47 @@ void iov_iter_pipe(struct iov_iter *i, unsigned int direction, { BUG_ON(direction != READ); WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size)); - i->type = ITER_PIPE | READ; - i->pipe = pipe; - i->head = pipe->head; - i->iov_offset = 0; - i->count = count; - i->start_head = i->head; + *i = (struct iov_iter){ + .iter_type = ITER_PIPE, + .data_source = false, + .pipe = pipe, + .head = pipe->head, + .start_head = pipe->head, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_pipe); /** + * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray + * @i: The iterator to initialise. + * @direction: The direction of the transfer. + * @xarray: The xarray to access. + * @start: The start file position. + * @count: The size of the I/O buffer in bytes. + * + * Set up an I/O iterator to either draw data out of the pages attached to an + * inode or to inject data into those pages. The pages *must* be prevented + * from evaporation, either by taking a ref on them or locking them by the + * caller. + */ +void iov_iter_xarray(struct iov_iter *i, unsigned int direction, + struct xarray *xarray, loff_t start, size_t count) +{ + BUG_ON(direction & ~1); + *i = (struct iov_iter) { + .iter_type = ITER_XARRAY, + .data_source = direction, + .xarray = xarray, + .xarray_start = start, + .count = count, + .iov_offset = 0 + }; +} +EXPORT_SYMBOL(iov_iter_xarray); + +/** * iov_iter_discard - Initialise an I/O iterator that discards data * @i: The iterator to initialise. * @direction: The direction of the transfer. @@ -1214,51 +1259,103 @@ EXPORT_SYMBOL(iov_iter_pipe); void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) { BUG_ON(direction != READ); - i->type = ITER_DISCARD | READ; - i->count = count; - i->iov_offset = 0; + *i = (struct iov_iter){ + .iter_type = ITER_DISCARD, + .data_source = false, + .count = count, + .iov_offset = 0 + }; } EXPORT_SYMBOL(iov_iter_discard); -unsigned long iov_iter_alignment(const struct iov_iter *i) +static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) { unsigned long res = 0; size_t size = i->count; + size_t skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + size_t len = i->iov[k].iov_len - skip; + if (len) { + res |= (unsigned long)i->iov[k].iov_base + skip; + if (len > size) + len = size; + res |= len; + size -= len; + if (!size) + break; + } + } + return res; +} + +static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i) +{ + unsigned res = 0; + size_t size = i->count; + unsigned skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + size_t len = i->bvec[k].bv_len - skip; + res |= (unsigned long)i->bvec[k].bv_offset + skip; + if (len > size) + len = size; + res |= len; + size -= len; + if (!size) + break; + } + return res; +} - if (unlikely(iov_iter_is_pipe(i))) { +unsigned long iov_iter_alignment(const struct iov_iter *i) +{ + /* iovec and kvec have identical layouts */ + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return iov_iter_alignment_iovec(i); + + if (iov_iter_is_bvec(i)) + return iov_iter_alignment_bvec(i); + + if (iov_iter_is_pipe(i)) { unsigned int p_mask = i->pipe->ring_size - 1; + size_t size = i->count; if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) return size | i->iov_offset; return size; } - iterate_all_kinds(i, size, v, - (res |= (unsigned long)v.iov_base | v.iov_len, 0), - res |= v.bv_offset | v.bv_len, - res |= (unsigned long)v.iov_base | v.iov_len - ) - return res; + + if (iov_iter_is_xarray(i)) + return (i->xarray_start + i->iov_offset) | i->count; + + return 0; } EXPORT_SYMBOL(iov_iter_alignment); unsigned long iov_iter_gap_alignment(const struct iov_iter *i) { unsigned long res = 0; + unsigned long v = 0; size_t size = i->count; + unsigned k; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); + if (WARN_ON(!iter_is_iovec(i))) return ~0U; - } - iterate_all_kinds(i, size, v, - (res |= (!res ? 0 : (unsigned long)v.iov_base) | - (size != v.iov_len ? size : 0), 0), - (res |= (!res ? 0 : (unsigned long)v.bv_offset) | - (size != v.bv_len ? size : 0)), - (res |= (!res ? 0 : (unsigned long)v.iov_base) | - (size != v.iov_len ? size : 0)) - ); + for (k = 0; k < i->nr_segs; k++) { + if (i->iov[k].iov_len) { + unsigned long base = (unsigned long)i->iov[k].iov_base; + if (v) // if not the first one + res |= base | v; // this start | previous end + v = base + i->iov[k].iov_len; + if (size <= i->iov[k].iov_len) + break; + size -= i->iov[k].iov_len; + } + } return res; } EXPORT_SYMBOL(iov_iter_gap_alignment); @@ -1293,9 +1390,6 @@ static ssize_t pipe_get_pages(struct iov_iter *i, unsigned int iter_head, npages; size_t capacity; - if (!maxsize) - return 0; - if (!sanity(i)) return -EFAULT; @@ -1307,44 +1401,162 @@ static ssize_t pipe_get_pages(struct iov_iter *i, return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start); } +static ssize_t iter_xarray_populate_pages(struct page **pages, struct xarray *xa, + pgoff_t index, unsigned int nr_pages) +{ + XA_STATE(xas, xa, index); + struct page *page; + unsigned int ret = 0; + + rcu_read_lock(); + for (page = xas_load(&xas); page; page = xas_next(&xas)) { + if (xas_retry(&xas, page)) + continue; + + /* Has the page moved or been split? */ + if (unlikely(page != xas_reload(&xas))) { + xas_reset(&xas); + continue; + } + + pages[ret] = find_subpage(page, xas.xa_index); + get_page(pages[ret]); + if (++ret == nr_pages) + break; + } + rcu_read_unlock(); + return ret; +} + +static ssize_t iter_xarray_get_pages(struct iov_iter *i, + struct page **pages, size_t maxsize, + unsigned maxpages, size_t *_start_offset) +{ + unsigned nr, offset; + pgoff_t index, count; + size_t size = maxsize, actual; + loff_t pos; + + if (!size || !maxpages) + return 0; + + pos = i->xarray_start + i->iov_offset; + index = pos >> PAGE_SHIFT; + offset = pos & ~PAGE_MASK; + *_start_offset = offset; + + count = 1; + if (size > PAGE_SIZE - offset) { + size -= PAGE_SIZE - offset; + count += size >> PAGE_SHIFT; + size &= ~PAGE_MASK; + if (size) + count++; + } + + if (count > maxpages) + count = maxpages; + + nr = iter_xarray_populate_pages(pages, i->xarray, index, count); + if (nr == 0) + return 0; + + actual = PAGE_SIZE * nr; + actual -= offset; + if (nr == count && size > 0) { + unsigned last_offset = (nr > 1) ? 0 : offset; + actual -= PAGE_SIZE - (last_offset + size); + } + return actual; +} + +/* must be done on non-empty ITER_IOVEC one */ +static unsigned long first_iovec_segment(const struct iov_iter *i, + size_t *size, size_t *start, + size_t maxsize, unsigned maxpages) +{ + size_t skip; + long k; + + for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) { + unsigned long addr = (unsigned long)i->iov[k].iov_base + skip; + size_t len = i->iov[k].iov_len - skip; + + if (unlikely(!len)) + continue; + if (len > maxsize) + len = maxsize; + len += (*start = addr % PAGE_SIZE); + if (len > maxpages * PAGE_SIZE) + len = maxpages * PAGE_SIZE; + *size = len; + return addr & PAGE_MASK; + } + BUG(); // if it had been empty, we wouldn't get called +} + +/* must be done on non-empty ITER_BVEC one */ +static struct page *first_bvec_segment(const struct iov_iter *i, + size_t *size, size_t *start, + size_t maxsize, unsigned maxpages) +{ + struct page *page; + size_t skip = i->iov_offset, len; + + len = i->bvec->bv_len - skip; + if (len > maxsize) + len = maxsize; + skip += i->bvec->bv_offset; + page = i->bvec->bv_page + skip / PAGE_SIZE; + len += (*start = skip % PAGE_SIZE); + if (len > maxpages * PAGE_SIZE) + len = maxpages * PAGE_SIZE; + *size = len; + return page; +} + ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages, size_t maxsize, unsigned maxpages, size_t *start) { + size_t len; + int n, res; + if (maxsize > i->count) maxsize = i->count; + if (!maxsize) + return 0; - if (unlikely(iov_iter_is_pipe(i))) - return pipe_get_pages(i, pages, maxsize, maxpages, start); - if (unlikely(iov_iter_is_discard(i))) - return -EFAULT; + if (likely(iter_is_iovec(i))) { + unsigned int gup_flags = 0; + unsigned long addr; - iterate_all_kinds(i, maxsize, v, ({ - unsigned long addr = (unsigned long)v.iov_base; - size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); - int n; - int res; + if (iov_iter_rw(i) != WRITE) + gup_flags |= FOLL_WRITE; + if (i->nofault) + gup_flags |= FOLL_NOFAULT; - if (len > maxpages * PAGE_SIZE) - len = maxpages * PAGE_SIZE; - addr &= ~(PAGE_SIZE - 1); + addr = first_iovec_segment(i, &len, start, maxsize, maxpages); n = DIV_ROUND_UP(len, PAGE_SIZE); - res = get_user_pages_fast(addr, n, - iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, - pages); - if (unlikely(res < 0)) + res = get_user_pages_fast(addr, n, gup_flags, pages); + if (unlikely(res <= 0)) return res; return (res == n ? len : res * PAGE_SIZE) - *start; - 0;}),({ - /* can't be more than PAGE_SIZE */ - *start = v.bv_offset; - get_page(*pages = v.bv_page); - return v.bv_len; - }),({ - return -EFAULT; - }) - ) - return 0; + } + if (iov_iter_is_bvec(i)) { + struct page *page; + + page = first_bvec_segment(i, &len, start, maxsize, maxpages); + n = DIV_ROUND_UP(len, PAGE_SIZE); + while (n--) + get_page(*pages++ = page++); + return len - *start; + } + if (iov_iter_is_pipe(i)) + return pipe_get_pages(i, pages, maxsize, maxpages, start); + if (iov_iter_is_xarray(i)) + return iter_xarray_get_pages(i, pages, maxsize, maxpages, start); + return -EFAULT; } EXPORT_SYMBOL(iov_iter_get_pages); @@ -1361,9 +1573,6 @@ static ssize_t pipe_get_pages_alloc(struct iov_iter *i, unsigned int iter_head, npages; ssize_t n; - if (!maxsize) - return 0; - if (!sanity(i)) return -EFAULT; @@ -1386,88 +1595,122 @@ static ssize_t pipe_get_pages_alloc(struct iov_iter *i, return n; } +static ssize_t iter_xarray_get_pages_alloc(struct iov_iter *i, + struct page ***pages, size_t maxsize, + size_t *_start_offset) +{ + struct page **p; + unsigned nr, offset; + pgoff_t index, count; + size_t size = maxsize, actual; + loff_t pos; + + if (!size) + return 0; + + pos = i->xarray_start + i->iov_offset; + index = pos >> PAGE_SHIFT; + offset = pos & ~PAGE_MASK; + *_start_offset = offset; + + count = 1; + if (size > PAGE_SIZE - offset) { + size -= PAGE_SIZE - offset; + count += size >> PAGE_SHIFT; + size &= ~PAGE_MASK; + if (size) + count++; + } + + p = get_pages_array(count); + if (!p) + return -ENOMEM; + *pages = p; + + nr = iter_xarray_populate_pages(p, i->xarray, index, count); + if (nr == 0) + return 0; + + actual = PAGE_SIZE * nr; + actual -= offset; + if (nr == count && size > 0) { + unsigned last_offset = (nr > 1) ? 0 : offset; + actual -= PAGE_SIZE - (last_offset + size); + } + return actual; +} + ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages, size_t maxsize, size_t *start) { struct page **p; + size_t len; + int n, res; if (maxsize > i->count) maxsize = i->count; + if (!maxsize) + return 0; - if (unlikely(iov_iter_is_pipe(i))) - return pipe_get_pages_alloc(i, pages, maxsize, start); - if (unlikely(iov_iter_is_discard(i))) - return -EFAULT; + if (likely(iter_is_iovec(i))) { + unsigned int gup_flags = 0; + unsigned long addr; - iterate_all_kinds(i, maxsize, v, ({ - unsigned long addr = (unsigned long)v.iov_base; - size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); - int n; - int res; + if (iov_iter_rw(i) != WRITE) + gup_flags |= FOLL_WRITE; + if (i->nofault) + gup_flags |= FOLL_NOFAULT; - addr &= ~(PAGE_SIZE - 1); + addr = first_iovec_segment(i, &len, start, maxsize, ~0U); n = DIV_ROUND_UP(len, PAGE_SIZE); p = get_pages_array(n); if (!p) return -ENOMEM; - res = get_user_pages_fast(addr, n, - iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, p); - if (unlikely(res < 0)) { + res = get_user_pages_fast(addr, n, gup_flags, p); + if (unlikely(res <= 0)) { kvfree(p); + *pages = NULL; return res; } *pages = p; return (res == n ? len : res * PAGE_SIZE) - *start; - 0;}),({ - /* can't be more than PAGE_SIZE */ - *start = v.bv_offset; - *pages = p = get_pages_array(1); + } + if (iov_iter_is_bvec(i)) { + struct page *page; + + page = first_bvec_segment(i, &len, start, maxsize, ~0U); + n = DIV_ROUND_UP(len, PAGE_SIZE); + *pages = p = get_pages_array(n); if (!p) return -ENOMEM; - get_page(*p = v.bv_page); - return v.bv_len; - }),({ - return -EFAULT; - }) - ) - return 0; + while (n--) + get_page(*p++ = page++); + return len - *start; + } + if (iov_iter_is_pipe(i)) + return pipe_get_pages_alloc(i, pages, maxsize, start); + if (iov_iter_is_xarray(i)) + return iter_xarray_get_pages_alloc(i, pages, maxsize, start); + return -EFAULT; } EXPORT_SYMBOL(iov_iter_get_pages_alloc); size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i) { - char *to = addr; __wsum sum, next; - size_t off = 0; sum = *csum; if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, ({ - int err = 0; - next = csum_and_copy_from_user(v.iov_base, - (to += v.iov_len) - v.iov_len, - v.iov_len, 0, &err); - if (!err) { - sum = csum_block_add(sum, next, off); - off += v.iov_len; - } - err ? v.iov_len : 0; + iterate_and_advance(i, bytes, base, len, off, ({ + next = csum_and_copy_from_user(base, addr + off, len); + sum = csum_block_add(sum, next, off); + next ? 0 : len; }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ - sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len, - sum, off); - off += v.iov_len; + sum = csum_and_memcpy(addr + off, base, len, sum, off); }) ) *csum = sum; @@ -1475,90 +1718,30 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, } EXPORT_SYMBOL(csum_and_copy_from_iter); -bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, - struct iov_iter *i) -{ - char *to = addr; - __wsum sum, next; - size_t off = 0; - sum = *csum; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - iterate_all_kinds(i, bytes, v, ({ - int err = 0; - next = csum_and_copy_from_user(v.iov_base, - (to += v.iov_len) - v.iov_len, - v.iov_len, 0, &err); - if (err) - return false; - sum = csum_block_add(sum, next, off); - off += v.iov_len; - 0; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ - sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len, - sum, off); - off += v.iov_len; - }) - ) - *csum = sum; - iov_iter_advance(i, bytes); - return true; -} -EXPORT_SYMBOL(csum_and_copy_from_iter_full); - -size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, +size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, struct iov_iter *i) { - const char *from = addr; - __wsum *csum = csump; + struct csum_state *csstate = _csstate; __wsum sum, next; - size_t off = 0; - if (unlikely(iov_iter_is_pipe(i))) - return csum_and_copy_to_pipe_iter(addr, bytes, csum, i); - - sum = *csum; if (unlikely(iov_iter_is_discard(i))) { WARN_ON(1); /* for now */ return 0; } - iterate_and_advance(i, bytes, v, ({ - int err = 0; - next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len, - v.iov_base, - v.iov_len, 0, &err); - if (!err) { - sum = csum_block_add(sum, next, off); - off += v.iov_len; - } - err ? v.iov_len : 0; + + sum = csum_shift(csstate->csum, csstate->off); + if (unlikely(iov_iter_is_pipe(i))) + bytes = csum_and_copy_to_pipe_iter(addr, bytes, i, &sum); + else iterate_and_advance(i, bytes, base, len, off, ({ + next = csum_and_copy_to_user(addr + off, base, len); + sum = csum_block_add(sum, next, off); + next ? 0 : len; }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy(p + v.bv_offset, - (from += v.bv_len) - v.bv_len, - v.bv_len, sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ - sum = csum_and_memcpy(v.iov_base, - (from += v.iov_len) - v.iov_len, - v.iov_len, sum, off); - off += v.iov_len; + sum = csum_and_memcpy(base, addr + off, len, sum, off); }) ) - *csum = sum; + csstate->csum = csum_shift(sum, csstate->off); + csstate->off += bytes; return bytes; } EXPORT_SYMBOL(csum_and_copy_to_iter); @@ -1566,7 +1749,7 @@ EXPORT_SYMBOL(csum_and_copy_to_iter); size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, struct iov_iter *i) { -#ifdef CONFIG_CRYPTO +#ifdef CONFIG_CRYPTO_HASH struct ahash_request *hash = hashp; struct scatterlist sg; size_t copied; @@ -1582,19 +1765,56 @@ size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, } EXPORT_SYMBOL(hash_and_copy_to_iter); -int iov_iter_npages(const struct iov_iter *i, int maxpages) +static int iov_npages(const struct iov_iter *i, int maxpages) { - size_t size = i->count; + size_t skip = i->iov_offset, size = i->count; + const struct iovec *p; int npages = 0; - if (!size) - return 0; - if (unlikely(iov_iter_is_discard(i))) - return 0; + for (p = i->iov; size; skip = 0, p++) { + unsigned offs = offset_in_page(p->iov_base + skip); + size_t len = min(p->iov_len - skip, size); - if (unlikely(iov_iter_is_pipe(i))) { - struct pipe_inode_info *pipe = i->pipe; + if (len) { + size -= len; + npages += DIV_ROUND_UP(offs + len, PAGE_SIZE); + if (unlikely(npages > maxpages)) + return maxpages; + } + } + return npages; +} + +static int bvec_npages(const struct iov_iter *i, int maxpages) +{ + size_t skip = i->iov_offset, size = i->count; + const struct bio_vec *p; + int npages = 0; + + for (p = i->bvec; size; skip = 0, p++) { + unsigned offs = (p->bv_offset + skip) % PAGE_SIZE; + size_t len = min(p->bv_len - skip, size); + + size -= len; + npages += DIV_ROUND_UP(offs + len, PAGE_SIZE); + if (unlikely(npages > maxpages)) + return maxpages; + } + return npages; +} + +int iov_iter_npages(const struct iov_iter *i, int maxpages) +{ + if (unlikely(!i->count)) + return 0; + /* iovec and kvec have identical layouts */ + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return iov_npages(i, maxpages); + if (iov_iter_is_bvec(i)) + return bvec_npages(i, maxpages); + if (iov_iter_is_pipe(i)) { unsigned int iter_head; + int npages; size_t off; if (!sanity(i)) @@ -1602,28 +1822,15 @@ int iov_iter_npages(const struct iov_iter *i, int maxpages) data_start(i, &iter_head, &off); /* some of this one + all after this one */ - npages = pipe_space_for_user(iter_head, pipe->tail, pipe); - if (npages >= maxpages) - return maxpages; - } else iterate_all_kinds(i, size, v, ({ - unsigned long p = (unsigned long)v.iov_base; - npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) - - p / PAGE_SIZE; - if (npages >= maxpages) - return maxpages; - 0;}),({ - npages++; - if (npages >= maxpages) - return maxpages; - }),({ - unsigned long p = (unsigned long)v.iov_base; - npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) - - p / PAGE_SIZE; - if (npages >= maxpages) - return maxpages; - }) - ) - return npages; + npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); + return min(npages, maxpages); + } + if (iov_iter_is_xarray(i)) { + unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE; + int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE); + return min(npages, maxpages); + } + return 0; } EXPORT_SYMBOL(iov_iter_npages); @@ -1634,7 +1841,7 @@ const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) WARN_ON(1); return NULL; } - if (unlikely(iov_iter_is_discard(new))) + if (unlikely(iov_iter_is_discard(new) || iov_iter_is_xarray(new))) return NULL; if (iov_iter_is_bvec(new)) return new->bvec = kmemdup(new->bvec, @@ -1648,16 +1855,145 @@ const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) } EXPORT_SYMBOL(dup_iter); +static int copy_compat_iovec_from_user(struct iovec *iov, + const struct iovec __user *uvec, unsigned long nr_segs) +{ + const struct compat_iovec __user *uiov = + (const struct compat_iovec __user *)uvec; + int ret = -EFAULT, i; + + if (!user_access_begin(uiov, nr_segs * sizeof(*uiov))) + return -EFAULT; + + for (i = 0; i < nr_segs; i++) { + compat_uptr_t buf; + compat_ssize_t len; + + unsafe_get_user(len, &uiov[i].iov_len, uaccess_end); + unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end); + + /* check for compat_size_t not fitting in compat_ssize_t .. */ + if (len < 0) { + ret = -EINVAL; + goto uaccess_end; + } + iov[i].iov_base = compat_ptr(buf); + iov[i].iov_len = len; + } + + ret = 0; +uaccess_end: + user_access_end(); + return ret; +} + +static int copy_iovec_from_user(struct iovec *iov, + const struct iovec __user *uvec, unsigned long nr_segs) +{ + unsigned long seg; + + if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec))) + return -EFAULT; + for (seg = 0; seg < nr_segs; seg++) { + if ((ssize_t)iov[seg].iov_len < 0) + return -EINVAL; + } + + return 0; +} + +struct iovec *iovec_from_user(const struct iovec __user *uvec, + unsigned long nr_segs, unsigned long fast_segs, + struct iovec *fast_iov, bool compat) +{ + struct iovec *iov = fast_iov; + int ret; + + /* + * SuS says "The readv() function *may* fail if the iovcnt argument was + * less than or equal to 0, or greater than {IOV_MAX}. Linux has + * traditionally returned zero for zero segments, so... + */ + if (nr_segs == 0) + return iov; + if (nr_segs > UIO_MAXIOV) + return ERR_PTR(-EINVAL); + if (nr_segs > fast_segs) { + iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL); + if (!iov) + return ERR_PTR(-ENOMEM); + } + + if (compat) + ret = copy_compat_iovec_from_user(iov, uvec, nr_segs); + else + ret = copy_iovec_from_user(iov, uvec, nr_segs); + if (ret) { + if (iov != fast_iov) + kfree(iov); + return ERR_PTR(ret); + } + + return iov; +} + +ssize_t __import_iovec(int type, const struct iovec __user *uvec, + unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, + struct iov_iter *i, bool compat) +{ + ssize_t total_len = 0; + unsigned long seg; + struct iovec *iov; + + iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat); + if (IS_ERR(iov)) { + *iovp = NULL; + return PTR_ERR(iov); + } + + /* + * According to the Single Unix Specification we should return EINVAL if + * an element length is < 0 when cast to ssize_t or if the total length + * would overflow the ssize_t return value of the system call. + * + * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the + * overflow case. + */ + for (seg = 0; seg < nr_segs; seg++) { + ssize_t len = (ssize_t)iov[seg].iov_len; + + if (!access_ok(iov[seg].iov_base, len)) { + if (iov != *iovp) + kfree(iov); + *iovp = NULL; + return -EFAULT; + } + + if (len > MAX_RW_COUNT - total_len) { + len = MAX_RW_COUNT - total_len; + iov[seg].iov_len = len; + } + total_len += len; + } + + iov_iter_init(i, type, iov, nr_segs, total_len); + if (iov == *iovp) + *iovp = NULL; + else + *iovp = iov; + return total_len; +} + /** * import_iovec() - Copy an array of &struct iovec from userspace * into the kernel, check that it is valid, and initialize a new * &struct iov_iter iterator to access it. * * @type: One of %READ or %WRITE. - * @uvector: Pointer to the userspace array. + * @uvec: Pointer to the userspace array. * @nr_segs: Number of elements in userspace array. * @fast_segs: Number of elements in @iov. - * @iov: (input and output parameter) Pointer to pointer to (usually small + * @iovp: (input and output parameter) Pointer to pointer to (usually small * on-stack) kernel array. * @i: Pointer to iterator that will be initialized on success. * @@ -1670,51 +2006,15 @@ EXPORT_SYMBOL(dup_iter); * * Return: Negative error code on error, bytes imported on success */ -ssize_t import_iovec(int type, const struct iovec __user * uvector, +ssize_t import_iovec(int type, const struct iovec __user *uvec, unsigned nr_segs, unsigned fast_segs, - struct iovec **iov, struct iov_iter *i) + struct iovec **iovp, struct iov_iter *i) { - ssize_t n; - struct iovec *p; - n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, - *iov, &p); - if (n < 0) { - if (p != *iov) - kfree(p); - *iov = NULL; - return n; - } - iov_iter_init(i, type, p, nr_segs, n); - *iov = p == *iov ? NULL : p; - return n; + return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i, + in_compat_syscall()); } EXPORT_SYMBOL(import_iovec); -#ifdef CONFIG_COMPAT -#include <linux/compat.h> - -ssize_t compat_import_iovec(int type, - const struct compat_iovec __user * uvector, - unsigned nr_segs, unsigned fast_segs, - struct iovec **iov, struct iov_iter *i) -{ - ssize_t n; - struct iovec *p; - n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, - *iov, &p); - if (n < 0) { - if (p != *iov) - kfree(p); - *iov = NULL; - return n; - } - iov_iter_init(i, type, p, nr_segs, n); - *iov = p == *iov ? NULL : p; - return n; -} -EXPORT_SYMBOL(compat_import_iovec); -#endif - int import_single_range(int rw, void __user *buf, size_t len, struct iovec *iov, struct iov_iter *i) { @@ -1730,24 +2030,38 @@ int import_single_range(int rw, void __user *buf, size_t len, } EXPORT_SYMBOL(import_single_range); -int iov_iter_for_each_range(struct iov_iter *i, size_t bytes, - int (*f)(struct kvec *vec, void *context), - void *context) +/** + * iov_iter_restore() - Restore a &struct iov_iter to the same state as when + * iov_iter_save_state() was called. + * + * @i: &struct iov_iter to restore + * @state: state to restore from + * + * Used after iov_iter_save_state() to bring restore @i, if operations may + * have advanced it. + * + * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC + */ +void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) { - struct kvec w; - int err = -EINVAL; - if (!bytes) - return 0; - - iterate_all_kinds(i, bytes, v, -EINVAL, ({ - w.iov_base = kmap(v.bv_page) + v.bv_offset; - w.iov_len = v.bv_len; - err = f(&w, context); - kunmap(v.bv_page); - err;}), ({ - w = v; - err = f(&w, context);}) - ) - return err; + if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) && + !iov_iter_is_kvec(i)) + return; + i->iov_offset = state->iov_offset; + i->count = state->count; + /* + * For the *vec iters, nr_segs + iov is constant - if we increment + * the vec, then we also decrement the nr_segs count. Hence we don't + * need to track both of these, just one is enough and we can deduct + * the other from that. ITER_KVEC and ITER_IOVEC are the same struct + * size, so we can just increment the iov pointer as they are unionzed. + * ITER_BVEC _may_ be the same size on some archs, but on others it is + * not. Be safe and handle it separately. + */ + BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec)); + if (iov_iter_is_bvec(i)) + i->bvec -= state->nr_segs - i->nr_segs; + else + i->iov -= state->nr_segs - i->nr_segs; + i->nr_segs = state->nr_segs; } -EXPORT_SYMBOL(iov_iter_for_each_range); diff --git a/lib/kasprintf.c b/lib/kasprintf.c index bacf7b83ccf0..cd2f5974ed98 100644 --- a/lib/kasprintf.c +++ b/lib/kasprintf.c @@ -5,7 +5,7 @@ * Copyright (C) 1991, 1992 Linus Torvalds */ -#include <stdarg.h> +#include <linux/stdarg.h> #include <linux/export.h> #include <linux/slab.h> #include <linux/types.h> diff --git a/lib/kfifo.c b/lib/kfifo.c index 70dab9ac7827..12f5a347aa13 100644 --- a/lib/kfifo.c +++ b/lib/kfifo.c @@ -415,7 +415,7 @@ static unsigned int __kfifo_peek_n(struct __kfifo *fifo, size_t recsize) ) /* - * __kfifo_poke_n internal helper function for storeing the length of + * __kfifo_poke_n internal helper function for storing the length of * the record into the fifo */ static void __kfifo_poke_n(struct __kfifo *fifo, unsigned int n, size_t recsize) diff --git a/lib/kobject.c b/lib/kobject.c index 83198cb37d8d..4a56f519139d 100644 --- a/lib/kobject.c +++ b/lib/kobject.c @@ -6,7 +6,7 @@ * Copyright (c) 2006-2007 Greg Kroah-Hartman <greg@kroah.com> * Copyright (c) 2006-2007 Novell Inc. * - * Please see the file Documentation/kobject.txt for critical information + * Please see the file Documentation/core-api/kobject.rst for critical information * about using the kobject interface. */ @@ -599,35 +599,50 @@ out: } EXPORT_SYMBOL_GPL(kobject_move); -/** - * kobject_del() - Unlink kobject from hierarchy. - * @kobj: object. - * - * This is the function that should be called to delete an object - * successfully added via kobject_add(). - */ -void kobject_del(struct kobject *kobj) +static void __kobject_del(struct kobject *kobj) { struct kernfs_node *sd; const struct kobj_type *ktype; - if (!kobj) - return; - sd = kobj->sd; ktype = get_ktype(kobj); if (ktype) sysfs_remove_groups(kobj, ktype->default_groups); + /* send "remove" if the caller did not do it but sent "add" */ + if (kobj->state_add_uevent_sent && !kobj->state_remove_uevent_sent) { + pr_debug("kobject: '%s' (%p): auto cleanup 'remove' event\n", + kobject_name(kobj), kobj); + kobject_uevent(kobj, KOBJ_REMOVE); + } + sysfs_remove_dir(kobj); sysfs_put(sd); kobj->state_in_sysfs = 0; kobj_kset_leave(kobj); - kobject_put(kobj->parent); kobj->parent = NULL; } + +/** + * kobject_del() - Unlink kobject from hierarchy. + * @kobj: object. + * + * This is the function that should be called to delete an object + * successfully added via kobject_add(). + */ +void kobject_del(struct kobject *kobj) +{ + struct kobject *parent; + + if (!kobj) + return; + + parent = kobj->parent; + __kobject_del(kobj); + kobject_put(parent); +} EXPORT_SYMBOL(kobject_del); /** @@ -663,6 +678,7 @@ EXPORT_SYMBOL(kobject_get_unless_zero); */ static void kobject_cleanup(struct kobject *kobj) { + struct kobject *parent = kobj->parent; struct kobj_type *t = get_ktype(kobj); const char *name = kobj->name; @@ -670,21 +686,17 @@ static void kobject_cleanup(struct kobject *kobj) kobject_name(kobj), kobj, __func__, kobj->parent); if (t && !t->release) - pr_debug("kobject: '%s' (%p): does not have a release() function, it is broken and must be fixed. See Documentation/kobject.txt.\n", - kobject_name(kobj), kobj); - - /* send "remove" if the caller did not do it but sent "add" */ - if (kobj->state_add_uevent_sent && !kobj->state_remove_uevent_sent) { - pr_debug("kobject: '%s' (%p): auto cleanup 'remove' event\n", + pr_debug("kobject: '%s' (%p): does not have a release() function, it is broken and must be fixed. See Documentation/core-api/kobject.rst.\n", kobject_name(kobj), kobj); - kobject_uevent(kobj, KOBJ_REMOVE); - } /* remove from sysfs if the caller did not do it */ if (kobj->state_in_sysfs) { pr_debug("kobject: '%s' (%p): auto cleanup kobject_del\n", kobject_name(kobj), kobj); - kobject_del(kobj); + __kobject_del(kobj); + } else { + /* avoid dropping the parent reference unnecessarily */ + parent = NULL; } if (t && t->release) { @@ -698,6 +710,8 @@ static void kobject_cleanup(struct kobject *kobj) pr_debug("kobject: '%s': free name\n", name); kfree_const(name); } + + kobject_put(parent); } #ifdef CONFIG_DEBUG_KOBJECT_RELEASE @@ -763,7 +777,7 @@ static struct kobj_type dynamic_kobj_ktype = { * call to kobject_put() and not kfree(), as kobject_init() has * already been called on this structure. */ -struct kobject *kobject_create(void) +static struct kobject *kobject_create(void) { struct kobject *kobj; diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c index 7998affa45d4..c87d5b6a8a55 100644 --- a/lib/kobject_uevent.c +++ b/lib/kobject_uevent.c @@ -251,12 +251,13 @@ static int kobj_usermode_filter(struct kobject *kobj) static int init_uevent_argv(struct kobj_uevent_env *env, const char *subsystem) { + int buffer_size = sizeof(env->buf) - env->buflen; int len; - len = strlcpy(&env->buf[env->buflen], subsystem, - sizeof(env->buf) - env->buflen); - if (len >= (sizeof(env->buf) - env->buflen)) { - WARN(1, KERN_ERR "init_uevent_argv: buffer size too small\n"); + len = strlcpy(&env->buf[env->buflen], subsystem, buffer_size); + if (len >= buffer_size) { + pr_warn("init_uevent_argv: buffer size of %d too small, needed %d\n", + buffer_size, len); return -ENOMEM; } diff --git a/lib/kstrtox.c b/lib/kstrtox.c index 1006bf70bf74..059b8b00dc53 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -14,11 +14,12 @@ */ #include <linux/ctype.h> #include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/math64.h> #include <linux/export.h> +#include <linux/kstrtox.h> +#include <linux/math64.h> #include <linux/types.h> #include <linux/uaccess.h> + #include "kstrtox.h" const char *_parse_integer_fixup_radix(const char *s, unsigned int *base) @@ -39,20 +40,22 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base) /* * Convert non-negative integer string representation in explicitly given radix - * to an integer. + * to an integer. A maximum of max_chars characters will be converted. + * * Return number of characters consumed maybe or-ed with overflow bit. * If overflow occurs, result integer (incorrect) is still returned. * * Don't you dare use this function. */ -unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *p) +unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *p, + size_t max_chars) { unsigned long long res; unsigned int rv; res = 0; rv = 0; - while (1) { + while (max_chars--) { unsigned int c = *s; unsigned int lc = c | 0x20; /* don't tolower() this line */ unsigned int val; @@ -82,6 +85,11 @@ unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long return rv; } +unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *p) +{ + return _parse_integer_limit(s, base, p, INT_MAX); +} + static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res) { unsigned long long _res; @@ -115,8 +123,7 @@ static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res) * @res: Where to write the result of the conversion on success. * * Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error. - * Used as a replacement for the obsolete simple_strtoull. Return code must - * be checked. + * Preferred over simple_strtoull(). Return code must be checked. */ int kstrtoull(const char *s, unsigned int base, unsigned long long *res) { @@ -139,8 +146,7 @@ EXPORT_SYMBOL(kstrtoull); * @res: Where to write the result of the conversion on success. * * Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error. - * Used as a replacement for the obsolete simple_strtoull. Return code must - * be checked. + * Preferred over simple_strtoll(). Return code must be checked. */ int kstrtoll(const char *s, unsigned int base, long long *res) { @@ -211,8 +217,7 @@ EXPORT_SYMBOL(_kstrtol); * @res: Where to write the result of the conversion on success. * * Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error. - * Used as a replacement for the obsolete simple_strtoull. Return code must - * be checked. + * Preferred over simple_strtoul(). Return code must be checked. */ int kstrtouint(const char *s, unsigned int base, unsigned int *res) { @@ -242,8 +247,7 @@ EXPORT_SYMBOL(kstrtouint); * @res: Where to write the result of the conversion on success. * * Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error. - * Used as a replacement for the obsolete simple_strtoull. Return code must - * be checked. + * Preferred over simple_strtol(). Return code must be checked. */ int kstrtoint(const char *s, unsigned int base, int *res) { @@ -359,6 +363,7 @@ int kstrtobool(const char *s, bool *res) default: break; } + break; default: break; } diff --git a/lib/kstrtox.h b/lib/kstrtox.h index 3b4637bcd254..158c400ca865 100644 --- a/lib/kstrtox.h +++ b/lib/kstrtox.h @@ -4,6 +4,8 @@ #define KSTRTOX_OVERFLOW (1U << 31) const char *_parse_integer_fixup_radix(const char *s, unsigned int *base); +unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *res, + size_t max_chars); unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *res); #endif diff --git a/lib/kunit/.kunitconfig b/lib/kunit/.kunitconfig new file mode 100644 index 000000000000..9235b7d42d38 --- /dev/null +++ b/lib/kunit/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_KUNIT_TEST=y +CONFIG_KUNIT_EXAMPLE_TEST=y diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig index af37016bfdd4..0b5dfb001bac 100644 --- a/lib/kunit/Kconfig +++ b/lib/kunit/Kconfig @@ -3,7 +3,8 @@ # menuconfig KUNIT - bool "KUnit - Enable support for unit tests" + tristate "KUnit - Enable support for unit tests" + select GLOB if KUNIT=y help Enables support for kernel unit tests (KUnit), a lightweight unit testing and mocking framework for the Linux kernel. These tests are @@ -14,8 +15,18 @@ menuconfig KUNIT if KUNIT +config KUNIT_DEBUGFS + bool "KUnit - Enable /sys/kernel/debug/kunit debugfs representation" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + Enable debugfs representation for kunit. Currently this consists + of /sys/kernel/debug/kunit/<test_suite>/results files for each + test suite, which allow users to see results of the last test suite + run that occurred. + config KUNIT_TEST - bool "KUnit test for KUnit" + tristate "KUnit test for KUnit" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS help Enables the unit tests for the KUnit test framework. These tests test the KUnit test framework itself; the tests are both written using @@ -24,7 +35,8 @@ config KUNIT_TEST expected. config KUNIT_EXAMPLE_TEST - bool "Example test for KUnit" + tristate "Example test for KUnit" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS help Enables an example unit test that illustrates some of the basic features of KUnit. This test only exists to help new users understand @@ -33,4 +45,18 @@ config KUNIT_EXAMPLE_TEST is intended for curious hackers who would like to understand how to use KUnit for kernel development. +config KUNIT_ALL_TESTS + tristate "All KUnit tests with satisfied dependencies" + help + Enables all KUnit tests, if they can be enabled. + KUnit tests run during boot and output the results to the debug log + in TAP format (http://testanything.org/). Only useful for kernel devs + running the KUnit test harness, and not intended for inclusion into a + production build. + + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + endif # KUNIT diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile index 769d9402b5d3..c49f4ffb6273 100644 --- a/lib/kunit/Makefile +++ b/lib/kunit/Makefile @@ -1,9 +1,20 @@ -obj-$(CONFIG_KUNIT) += test.o \ +obj-$(CONFIG_KUNIT) += kunit.o + +kunit-objs += test.o \ string-stream.o \ assert.o \ - try-catch.o + try-catch.o \ + executor.o + +ifeq ($(CONFIG_KUNIT_DEBUGFS),y) +kunit-objs += debugfs.o +endif + +obj-$(CONFIG_KUNIT_TEST) += kunit-test.o -obj-$(CONFIG_KUNIT_TEST) += test-test.o \ - string-stream-test.o +# string-stream-test compiles built-in only. +ifeq ($(CONFIG_KUNIT_TEST),y) +obj-$(CONFIG_KUNIT_TEST) += string-stream-test.o +endif -obj-$(CONFIG_KUNIT_EXAMPLE_TEST) += example-test.o +obj-$(CONFIG_KUNIT_EXAMPLE_TEST) += kunit-example-test.o diff --git a/lib/kunit/assert.c b/lib/kunit/assert.c index 86013d4cf891..b972bda61c0c 100644 --- a/lib/kunit/assert.c +++ b/lib/kunit/assert.c @@ -6,6 +6,9 @@ * Author: Brendan Higgins <brendanhiggins@google.com> */ #include <kunit/assert.h> +#include <kunit/test.h> + +#include "string-stream.h" void kunit_base_assert_format(const struct kunit_assert *assert, struct string_stream *stream) @@ -22,8 +25,9 @@ void kunit_base_assert_format(const struct kunit_assert *assert, } string_stream_add(stream, "%s FAILED at %s:%d\n", - expect_or_assert, assert->file, assert->line); + expect_or_assert, assert->file, assert->line); } +EXPORT_SYMBOL_GPL(kunit_base_assert_format); void kunit_assert_print_msg(const struct kunit_assert *assert, struct string_stream *stream) @@ -31,6 +35,7 @@ void kunit_assert_print_msg(const struct kunit_assert *assert, if (assert->message.fmt) string_stream_add(stream, "\n%pV", &assert->message); } +EXPORT_SYMBOL_GPL(kunit_assert_print_msg); void kunit_fail_assert_format(const struct kunit_assert *assert, struct string_stream *stream) @@ -38,104 +43,164 @@ void kunit_fail_assert_format(const struct kunit_assert *assert, kunit_base_assert_format(assert, stream); string_stream_add(stream, "%pV", &assert->message); } +EXPORT_SYMBOL_GPL(kunit_fail_assert_format); void kunit_unary_assert_format(const struct kunit_assert *assert, struct string_stream *stream) { - struct kunit_unary_assert *unary_assert = container_of( - assert, struct kunit_unary_assert, assert); + struct kunit_unary_assert *unary_assert; + + unary_assert = container_of(assert, struct kunit_unary_assert, assert); kunit_base_assert_format(assert, stream); if (unary_assert->expected_true) string_stream_add(stream, - "\tExpected %s to be true, but is false\n", - unary_assert->condition); + KUNIT_SUBTEST_INDENT "Expected %s to be true, but is false\n", + unary_assert->condition); else string_stream_add(stream, - "\tExpected %s to be false, but is true\n", - unary_assert->condition); + KUNIT_SUBTEST_INDENT "Expected %s to be false, but is true\n", + unary_assert->condition); kunit_assert_print_msg(assert, stream); } +EXPORT_SYMBOL_GPL(kunit_unary_assert_format); void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert, struct string_stream *stream) { - struct kunit_ptr_not_err_assert *ptr_assert = container_of( - assert, struct kunit_ptr_not_err_assert, assert); + struct kunit_ptr_not_err_assert *ptr_assert; + + ptr_assert = container_of(assert, struct kunit_ptr_not_err_assert, + assert); kunit_base_assert_format(assert, stream); if (!ptr_assert->value) { string_stream_add(stream, - "\tExpected %s is not null, but is\n", - ptr_assert->text); + KUNIT_SUBTEST_INDENT "Expected %s is not null, but is\n", + ptr_assert->text); } else if (IS_ERR(ptr_assert->value)) { string_stream_add(stream, - "\tExpected %s is not error, but is: %ld\n", - ptr_assert->text, - PTR_ERR(ptr_assert->value)); + KUNIT_SUBTEST_INDENT "Expected %s is not error, but is: %ld\n", + ptr_assert->text, + PTR_ERR(ptr_assert->value)); } kunit_assert_print_msg(assert, stream); } +EXPORT_SYMBOL_GPL(kunit_ptr_not_err_assert_format); + +/* Checks if `text` is a literal representing `value`, e.g. "5" and 5 */ +static bool is_literal(struct kunit *test, const char *text, long long value, + gfp_t gfp) +{ + char *buffer; + int len; + bool ret; + + len = snprintf(NULL, 0, "%lld", value); + if (strlen(text) != len) + return false; + + buffer = kunit_kmalloc(test, len+1, gfp); + if (!buffer) + return false; + + snprintf(buffer, len+1, "%lld", value); + ret = strncmp(buffer, text, len) == 0; + + kunit_kfree(test, buffer); + return ret; +} void kunit_binary_assert_format(const struct kunit_assert *assert, struct string_stream *stream) { - struct kunit_binary_assert *binary_assert = container_of( - assert, struct kunit_binary_assert, assert); + struct kunit_binary_assert *binary_assert; + + binary_assert = container_of(assert, struct kunit_binary_assert, + assert); kunit_base_assert_format(assert, stream); string_stream_add(stream, - "\tExpected %s %s %s, but\n", - binary_assert->left_text, - binary_assert->operation, - binary_assert->right_text); - string_stream_add(stream, "\t\t%s == %lld\n", - binary_assert->left_text, - binary_assert->left_value); - string_stream_add(stream, "\t\t%s == %lld", - binary_assert->right_text, - binary_assert->right_value); + KUNIT_SUBTEST_INDENT "Expected %s %s %s, but\n", + binary_assert->left_text, + binary_assert->operation, + binary_assert->right_text); + if (!is_literal(stream->test, binary_assert->left_text, + binary_assert->left_value, stream->gfp)) + string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %lld\n", + binary_assert->left_text, + binary_assert->left_value); + if (!is_literal(stream->test, binary_assert->right_text, + binary_assert->right_value, stream->gfp)) + string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %lld", + binary_assert->right_text, + binary_assert->right_value); kunit_assert_print_msg(assert, stream); } +EXPORT_SYMBOL_GPL(kunit_binary_assert_format); void kunit_binary_ptr_assert_format(const struct kunit_assert *assert, struct string_stream *stream) { - struct kunit_binary_ptr_assert *binary_assert = container_of( - assert, struct kunit_binary_ptr_assert, assert); + struct kunit_binary_ptr_assert *binary_assert; + + binary_assert = container_of(assert, struct kunit_binary_ptr_assert, + assert); kunit_base_assert_format(assert, stream); string_stream_add(stream, - "\tExpected %s %s %s, but\n", - binary_assert->left_text, - binary_assert->operation, - binary_assert->right_text); - string_stream_add(stream, "\t\t%s == %pK\n", - binary_assert->left_text, - binary_assert->left_value); - string_stream_add(stream, "\t\t%s == %pK", - binary_assert->right_text, - binary_assert->right_value); + KUNIT_SUBTEST_INDENT "Expected %s %s %s, but\n", + binary_assert->left_text, + binary_assert->operation, + binary_assert->right_text); + string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %px\n", + binary_assert->left_text, + binary_assert->left_value); + string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %px", + binary_assert->right_text, + binary_assert->right_value); kunit_assert_print_msg(assert, stream); } +EXPORT_SYMBOL_GPL(kunit_binary_ptr_assert_format); + +/* Checks if KUNIT_EXPECT_STREQ() args were string literals. + * Note: `text` will have ""s where as `value` will not. + */ +static bool is_str_literal(const char *text, const char *value) +{ + int len; + + len = strlen(text); + if (len < 2) + return false; + if (text[0] != '\"' || text[len - 1] != '\"') + return false; + + return strncmp(text + 1, value, len - 2) == 0; +} void kunit_binary_str_assert_format(const struct kunit_assert *assert, struct string_stream *stream) { - struct kunit_binary_str_assert *binary_assert = container_of( - assert, struct kunit_binary_str_assert, assert); + struct kunit_binary_str_assert *binary_assert; + + binary_assert = container_of(assert, struct kunit_binary_str_assert, + assert); kunit_base_assert_format(assert, stream); string_stream_add(stream, - "\tExpected %s %s %s, but\n", - binary_assert->left_text, - binary_assert->operation, - binary_assert->right_text); - string_stream_add(stream, "\t\t%s == %s\n", - binary_assert->left_text, - binary_assert->left_value); - string_stream_add(stream, "\t\t%s == %s", - binary_assert->right_text, - binary_assert->right_value); + KUNIT_SUBTEST_INDENT "Expected %s %s %s, but\n", + binary_assert->left_text, + binary_assert->operation, + binary_assert->right_text); + if (!is_str_literal(binary_assert->left_text, binary_assert->left_value)) + string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == \"%s\"\n", + binary_assert->left_text, + binary_assert->left_value); + if (!is_str_literal(binary_assert->right_text, binary_assert->right_value)) + string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == \"%s\"", + binary_assert->right_text, + binary_assert->right_value); kunit_assert_print_msg(assert, stream); } +EXPORT_SYMBOL_GPL(kunit_binary_str_assert_format); diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c new file mode 100644 index 000000000000..b71db0abc12b --- /dev/null +++ b/lib/kunit/debugfs.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020, Oracle and/or its affiliates. + * Author: Alan Maguire <alan.maguire@oracle.com> + */ + +#include <linux/debugfs.h> +#include <linux/module.h> + +#include <kunit/test.h> + +#include "string-stream.h" + +#define KUNIT_DEBUGFS_ROOT "kunit" +#define KUNIT_DEBUGFS_RESULTS "results" + +/* + * Create a debugfs representation of test suites: + * + * Path Semantics + * /sys/kernel/debug/kunit/<testsuite>/results Show results of last run for + * testsuite + * + */ + +static struct dentry *debugfs_rootdir; + +void kunit_debugfs_cleanup(void) +{ + debugfs_remove_recursive(debugfs_rootdir); +} + +void kunit_debugfs_init(void) +{ + if (!debugfs_rootdir) + debugfs_rootdir = debugfs_create_dir(KUNIT_DEBUGFS_ROOT, NULL); +} + +static void debugfs_print_result(struct seq_file *seq, + struct kunit_suite *suite, + struct kunit_case *test_case) +{ + if (!test_case || !test_case->log) + return; + + seq_printf(seq, "%s", test_case->log); +} + +/* + * /sys/kernel/debug/kunit/<testsuite>/results shows all results for testsuite. + */ +static int debugfs_print_results(struct seq_file *seq, void *v) +{ + struct kunit_suite *suite = (struct kunit_suite *)seq->private; + bool success = kunit_suite_has_succeeded(suite); + struct kunit_case *test_case; + + if (!suite || !suite->log) + return 0; + + seq_printf(seq, "%s", suite->log); + + kunit_suite_for_each_test_case(suite, test_case) + debugfs_print_result(seq, suite, test_case); + + seq_printf(seq, "%s %d - %s\n", + kunit_status_to_ok_not_ok(success), 1, suite->name); + return 0; +} + +static int debugfs_release(struct inode *inode, struct file *file) +{ + return single_release(inode, file); +} + +static int debugfs_results_open(struct inode *inode, struct file *file) +{ + struct kunit_suite *suite; + + suite = (struct kunit_suite *)inode->i_private; + + return single_open(file, debugfs_print_results, suite); +} + +static const struct file_operations debugfs_results_fops = { + .open = debugfs_results_open, + .read = seq_read, + .llseek = seq_lseek, + .release = debugfs_release, +}; + +void kunit_debugfs_create_suite(struct kunit_suite *suite) +{ + struct kunit_case *test_case; + + /* Allocate logs before creating debugfs representation. */ + suite->log = kzalloc(KUNIT_LOG_SIZE, GFP_KERNEL); + kunit_suite_for_each_test_case(suite, test_case) + test_case->log = kzalloc(KUNIT_LOG_SIZE, GFP_KERNEL); + + suite->debugfs = debugfs_create_dir(suite->name, debugfs_rootdir); + + debugfs_create_file(KUNIT_DEBUGFS_RESULTS, S_IFREG | 0444, + suite->debugfs, + suite, &debugfs_results_fops); +} + +void kunit_debugfs_destroy_suite(struct kunit_suite *suite) +{ + struct kunit_case *test_case; + + debugfs_remove_recursive(suite->debugfs); + kfree(suite->log); + kunit_suite_for_each_test_case(suite, test_case) + kfree(test_case->log); +} diff --git a/lib/kunit/debugfs.h b/lib/kunit/debugfs.h new file mode 100644 index 000000000000..dcc7d7556107 --- /dev/null +++ b/lib/kunit/debugfs.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020, Oracle and/or its affiliates. + */ + +#ifndef _KUNIT_DEBUGFS_H +#define _KUNIT_DEBUGFS_H + +#include <kunit/test.h> + +#ifdef CONFIG_KUNIT_DEBUGFS + +void kunit_debugfs_create_suite(struct kunit_suite *suite); +void kunit_debugfs_destroy_suite(struct kunit_suite *suite); +void kunit_debugfs_init(void); +void kunit_debugfs_cleanup(void); + +#else + +static inline void kunit_debugfs_create_suite(struct kunit_suite *suite) { } + +static inline void kunit_debugfs_destroy_suite(struct kunit_suite *suite) { } + +static inline void kunit_debugfs_init(void) { } + +static inline void kunit_debugfs_cleanup(void) { } + +#endif /* CONFIG_KUNIT_DEBUGFS */ + +#endif /* _KUNIT_DEBUGFS_H */ diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c new file mode 100644 index 000000000000..22640c9ee819 --- /dev/null +++ b/lib/kunit/executor.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/reboot.h> +#include <kunit/test.h> +#include <linux/glob.h> +#include <linux/moduleparam.h> + +/* + * These symbols point to the .kunit_test_suites section and are defined in + * include/asm-generic/vmlinux.lds.h, and consequently must be extern. + */ +extern struct kunit_suite * const * const __kunit_suites_start[]; +extern struct kunit_suite * const * const __kunit_suites_end[]; + +#if IS_BUILTIN(CONFIG_KUNIT) + +static char *filter_glob_param; +static char *action_param; + +module_param_named(filter_glob, filter_glob_param, charp, 0); +MODULE_PARM_DESC(filter_glob, + "Filter which KUnit test suites/tests run at boot-time, e.g. list* or list*.*del_test"); +module_param_named(action, action_param, charp, 0); +MODULE_PARM_DESC(action, + "Changes KUnit executor behavior, valid values are:\n" + "<none>: run the tests like normal\n" + "'list' to list test names instead of running them.\n"); + +/* glob_match() needs NULL terminated strings, so we need a copy of filter_glob_param. */ +struct kunit_test_filter { + char *suite_glob; + char *test_glob; +}; + +/* Split "suite_glob.test_glob" into two. Assumes filter_glob is not empty. */ +static void kunit_parse_filter_glob(struct kunit_test_filter *parsed, + const char *filter_glob) +{ + const int len = strlen(filter_glob); + const char *period = strchr(filter_glob, '.'); + + if (!period) { + parsed->suite_glob = kzalloc(len + 1, GFP_KERNEL); + parsed->test_glob = NULL; + strcpy(parsed->suite_glob, filter_glob); + return; + } + + parsed->suite_glob = kzalloc(period - filter_glob + 1, GFP_KERNEL); + parsed->test_glob = kzalloc(len - (period - filter_glob) + 1, GFP_KERNEL); + + strncpy(parsed->suite_glob, filter_glob, period - filter_glob); + strncpy(parsed->test_glob, period + 1, len - (period - filter_glob)); +} + +/* Create a copy of suite with only tests that match test_glob. */ +static struct kunit_suite * +kunit_filter_tests(struct kunit_suite *const suite, const char *test_glob) +{ + int n = 0; + struct kunit_case *filtered, *test_case; + struct kunit_suite *copy; + + kunit_suite_for_each_test_case(suite, test_case) { + if (!test_glob || glob_match(test_glob, test_case->name)) + ++n; + } + + if (n == 0) + return NULL; + + /* Use memcpy to workaround copy->name being const. */ + copy = kmalloc(sizeof(*copy), GFP_KERNEL); + memcpy(copy, suite, sizeof(*copy)); + + filtered = kcalloc(n + 1, sizeof(*filtered), GFP_KERNEL); + + n = 0; + kunit_suite_for_each_test_case(suite, test_case) { + if (!test_glob || glob_match(test_glob, test_case->name)) + filtered[n++] = *test_case; + } + + copy->test_cases = filtered; + return copy; +} + +static char *kunit_shutdown; +core_param(kunit_shutdown, kunit_shutdown, charp, 0644); + +static struct kunit_suite * const * +kunit_filter_subsuite(struct kunit_suite * const * const subsuite, + struct kunit_test_filter *filter) +{ + int i, n = 0; + struct kunit_suite **filtered, *filtered_suite; + + n = 0; + for (i = 0; subsuite[i]; ++i) { + if (glob_match(filter->suite_glob, subsuite[i]->name)) + ++n; + } + + if (n == 0) + return NULL; + + filtered = kmalloc_array(n + 1, sizeof(*filtered), GFP_KERNEL); + if (!filtered) + return NULL; + + n = 0; + for (i = 0; subsuite[i] != NULL; ++i) { + if (!glob_match(filter->suite_glob, subsuite[i]->name)) + continue; + filtered_suite = kunit_filter_tests(subsuite[i], filter->test_glob); + if (filtered_suite) + filtered[n++] = filtered_suite; + } + filtered[n] = NULL; + + return filtered; +} + +struct suite_set { + struct kunit_suite * const * const *start; + struct kunit_suite * const * const *end; +}; + +static void kunit_free_subsuite(struct kunit_suite * const *subsuite) +{ + unsigned int i; + + for (i = 0; subsuite[i]; i++) + kfree(subsuite[i]); + + kfree(subsuite); +} + +static void kunit_free_suite_set(struct suite_set suite_set) +{ + struct kunit_suite * const * const *suites; + + for (suites = suite_set.start; suites < suite_set.end; suites++) + kunit_free_subsuite(*suites); + kfree(suite_set.start); +} + +static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, + const char *filter_glob) +{ + int i; + struct kunit_suite * const **copy, * const *filtered_subsuite; + struct suite_set filtered; + struct kunit_test_filter filter; + + const size_t max = suite_set->end - suite_set->start; + + copy = kmalloc_array(max, sizeof(*filtered.start), GFP_KERNEL); + filtered.start = copy; + if (!copy) { /* won't be able to run anything, return an empty set */ + filtered.end = copy; + return filtered; + } + + kunit_parse_filter_glob(&filter, filter_glob); + + for (i = 0; i < max; ++i) { + filtered_subsuite = kunit_filter_subsuite(suite_set->start[i], &filter); + if (filtered_subsuite) + *copy++ = filtered_subsuite; + } + filtered.end = copy; + + kfree(filter.suite_glob); + kfree(filter.test_glob); + return filtered; +} + +static void kunit_handle_shutdown(void) +{ + if (!kunit_shutdown) + return; + + if (!strcmp(kunit_shutdown, "poweroff")) + kernel_power_off(); + else if (!strcmp(kunit_shutdown, "halt")) + kernel_halt(); + else if (!strcmp(kunit_shutdown, "reboot")) + kernel_restart(NULL); + +} + +static void kunit_print_tap_header(struct suite_set *suite_set) +{ + struct kunit_suite * const * const *suites, * const *subsuite; + int num_of_suites = 0; + + for (suites = suite_set->start; suites < suite_set->end; suites++) + for (subsuite = *suites; *subsuite != NULL; subsuite++) + num_of_suites++; + + pr_info("TAP version 14\n"); + pr_info("1..%d\n", num_of_suites); +} + +static void kunit_exec_run_tests(struct suite_set *suite_set) +{ + struct kunit_suite * const * const *suites; + + kunit_print_tap_header(suite_set); + + for (suites = suite_set->start; suites < suite_set->end; suites++) + __kunit_test_suites_init(*suites); +} + +static void kunit_exec_list_tests(struct suite_set *suite_set) +{ + unsigned int i; + struct kunit_suite * const * const *suites; + struct kunit_case *test_case; + + /* Hack: print a tap header so kunit.py can find the start of KUnit output. */ + pr_info("TAP version 14\n"); + + for (suites = suite_set->start; suites < suite_set->end; suites++) + for (i = 0; (*suites)[i] != NULL; i++) { + kunit_suite_for_each_test_case((*suites)[i], test_case) { + pr_info("%s.%s\n", (*suites)[i]->name, test_case->name); + } + } +} + +int kunit_run_all_tests(void) +{ + struct suite_set suite_set = { + .start = __kunit_suites_start, + .end = __kunit_suites_end, + }; + + if (filter_glob_param) + suite_set = kunit_filter_suites(&suite_set, filter_glob_param); + + if (!action_param) + kunit_exec_run_tests(&suite_set); + else if (strcmp(action_param, "list") == 0) + kunit_exec_list_tests(&suite_set); + else + pr_err("kunit executor: unknown action '%s'\n", action_param); + + if (filter_glob_param) { /* a copy was made of each array */ + kunit_free_suite_set(suite_set); + } + + kunit_handle_shutdown(); + + return 0; +} + +#if IS_BUILTIN(CONFIG_KUNIT_TEST) +#include "executor_test.c" +#endif + +#endif /* IS_BUILTIN(CONFIG_KUNIT) */ diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c new file mode 100644 index 000000000000..4ed57fd94e42 --- /dev/null +++ b/lib/kunit/executor_test.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for the KUnit executor. + * + * Copyright (C) 2021, Google LLC. + * Author: Daniel Latypov <dlatypov@google.com> + */ + +#include <kunit/test.h> + +static void kfree_at_end(struct kunit *test, const void *to_free); +static void free_subsuite_at_end(struct kunit *test, + struct kunit_suite *const *to_free); +static struct kunit_suite *alloc_fake_suite(struct kunit *test, + const char *suite_name, + struct kunit_case *test_cases); + +static void dummy_test(struct kunit *test) {} + +static struct kunit_case dummy_test_cases[] = { + /* .run_case is not important, just needs to be non-NULL */ + { .name = "test1", .run_case = dummy_test }, + { .name = "test2", .run_case = dummy_test }, + {}, +}; + +static void parse_filter_test(struct kunit *test) +{ + struct kunit_test_filter filter = {NULL, NULL}; + + kunit_parse_filter_glob(&filter, "suite"); + KUNIT_EXPECT_STREQ(test, filter.suite_glob, "suite"); + KUNIT_EXPECT_FALSE(test, filter.test_glob); + kfree(filter.suite_glob); + kfree(filter.test_glob); + + kunit_parse_filter_glob(&filter, "suite.test"); + KUNIT_EXPECT_STREQ(test, filter.suite_glob, "suite"); + KUNIT_EXPECT_STREQ(test, filter.test_glob, "test"); + kfree(filter.suite_glob); + kfree(filter.test_glob); +} + +static void filter_subsuite_test(struct kunit *test) +{ + struct kunit_suite *subsuite[3] = {NULL, NULL, NULL}; + struct kunit_suite * const *filtered; + struct kunit_test_filter filter = { + .suite_glob = "suite2", + .test_glob = NULL, + }; + + subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases); + subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases); + + /* Want: suite1, suite2, NULL -> suite2, NULL */ + filtered = kunit_filter_subsuite(subsuite, &filter); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered); + free_subsuite_at_end(test, filtered); + + /* Validate we just have suite2 */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]); + KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->name, "suite2"); + KUNIT_EXPECT_FALSE(test, filtered[1]); +} + +static void filter_subsuite_test_glob_test(struct kunit *test) +{ + struct kunit_suite *subsuite[3] = {NULL, NULL, NULL}; + struct kunit_suite * const *filtered; + struct kunit_test_filter filter = { + .suite_glob = "suite2", + .test_glob = "test2", + }; + + subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases); + subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases); + + /* Want: suite1, suite2, NULL -> suite2 (just test1), NULL */ + filtered = kunit_filter_subsuite(subsuite, &filter); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered); + free_subsuite_at_end(test, filtered); + + /* Validate we just have suite2 */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]); + KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->name, "suite2"); + KUNIT_EXPECT_FALSE(test, filtered[1]); + + /* Now validate we just have test2 */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]->test_cases); + KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->test_cases[0].name, "test2"); + KUNIT_EXPECT_FALSE(test, filtered[0]->test_cases[1].name); +} + +static void filter_subsuite_to_empty_test(struct kunit *test) +{ + struct kunit_suite *subsuite[3] = {NULL, NULL, NULL}; + struct kunit_suite * const *filtered; + struct kunit_test_filter filter = { + .suite_glob = "not_found", + .test_glob = NULL, + }; + + subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases); + subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases); + + filtered = kunit_filter_subsuite(subsuite, &filter); + free_subsuite_at_end(test, filtered); /* just in case */ + + KUNIT_EXPECT_FALSE_MSG(test, filtered, + "should be NULL to indicate no match"); +} + +static void kfree_subsuites_at_end(struct kunit *test, struct suite_set *suite_set) +{ + struct kunit_suite * const * const *suites; + + kfree_at_end(test, suite_set->start); + for (suites = suite_set->start; suites < suite_set->end; suites++) + free_subsuite_at_end(test, *suites); +} + +static void filter_suites_test(struct kunit *test) +{ + /* Suites per-file are stored as a NULL terminated array */ + struct kunit_suite *subsuites[2][2] = { + {NULL, NULL}, + {NULL, NULL}, + }; + /* Match the memory layout of suite_set */ + struct kunit_suite * const * const suites[2] = { + subsuites[0], subsuites[1], + }; + + const struct suite_set suite_set = { + .start = suites, + .end = suites + 2, + }; + struct suite_set filtered = {.start = NULL, .end = NULL}; + + /* Emulate two files, each having one suite */ + subsuites[0][0] = alloc_fake_suite(test, "suite0", dummy_test_cases); + subsuites[1][0] = alloc_fake_suite(test, "suite1", dummy_test_cases); + + /* Filter out suite1 */ + filtered = kunit_filter_suites(&suite_set, "suite0"); + kfree_subsuites_at_end(test, &filtered); /* let us use ASSERTs without leaking */ + KUNIT_ASSERT_EQ(test, filtered.end - filtered.start, (ptrdiff_t)1); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start[0]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start[0][0]); + KUNIT_EXPECT_STREQ(test, (const char *)filtered.start[0][0]->name, "suite0"); +} + +static struct kunit_case executor_test_cases[] = { + KUNIT_CASE(parse_filter_test), + KUNIT_CASE(filter_subsuite_test), + KUNIT_CASE(filter_subsuite_test_glob_test), + KUNIT_CASE(filter_subsuite_to_empty_test), + KUNIT_CASE(filter_suites_test), + {} +}; + +static struct kunit_suite executor_test_suite = { + .name = "kunit_executor_test", + .test_cases = executor_test_cases, +}; + +kunit_test_suites(&executor_test_suite); + +/* Test helpers */ + +static void kfree_res_free(struct kunit_resource *res) +{ + kfree(res->data); +} + +/* Use the resource API to register a call to kfree(to_free). + * Since we never actually use the resource, it's safe to use on const data. + */ +static void kfree_at_end(struct kunit *test, const void *to_free) +{ + /* kfree() handles NULL already, but avoid allocating a no-op cleanup. */ + if (IS_ERR_OR_NULL(to_free)) + return; + kunit_alloc_resource(test, NULL, kfree_res_free, GFP_KERNEL, + (void *)to_free); +} + +static void free_subsuite_res_free(struct kunit_resource *res) +{ + kunit_free_subsuite(res->data); +} + +static void free_subsuite_at_end(struct kunit *test, + struct kunit_suite *const *to_free) +{ + if (IS_ERR_OR_NULL(to_free)) + return; + kunit_alloc_resource(test, NULL, free_subsuite_res_free, + GFP_KERNEL, (void *)to_free); +} + +static struct kunit_suite *alloc_fake_suite(struct kunit *test, + const char *suite_name, + struct kunit_case *test_cases) +{ + struct kunit_suite *suite; + + /* We normally never expect to allocate suites, hence the non-const cast. */ + suite = kunit_kzalloc(test, sizeof(*suite), GFP_KERNEL); + strncpy((char *)suite->name, suite_name, sizeof(suite->name) - 1); + suite->test_cases = test_cases; + + return suite; +} diff --git a/lib/kunit/example-test.c b/lib/kunit/kunit-example-test.c index f64a829aa441..51099b0ca29c 100644 --- a/lib/kunit/example-test.c +++ b/lib/kunit/kunit-example-test.c @@ -41,6 +41,35 @@ static int example_test_init(struct kunit *test) } /* + * This test should always be skipped. + */ +static void example_skip_test(struct kunit *test) +{ + /* This line should run */ + kunit_info(test, "You should not see a line below."); + + /* Skip (and abort) the test */ + kunit_skip(test, "this test should be skipped"); + + /* This line should not execute */ + KUNIT_FAIL(test, "You should not see this line."); +} + +/* + * This test should always be marked skipped. + */ +static void example_mark_skipped_test(struct kunit *test) +{ + /* This line should run */ + kunit_info(test, "You should see a line below."); + + /* Skip (but do not abort) the test */ + kunit_mark_skipped(test, "this test should be skipped"); + + /* This line should run */ + kunit_info(test, "You should see this line."); +} +/* * Here we make a list of all the test cases we want to add to the test suite * below. */ @@ -52,6 +81,8 @@ static struct kunit_case example_test_cases[] = { * test suite. */ KUNIT_CASE(example_simple_test), + KUNIT_CASE(example_skip_test), + KUNIT_CASE(example_mark_skipped_test), {} }; @@ -85,4 +116,6 @@ static struct kunit_suite example_test_suite = { * This registers the above test suite telling KUnit that this is a suite of * tests that need to be run. */ -kunit_test_suite(example_test_suite); +kunit_test_suites(&example_test_suite); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/kunit/test-test.c b/lib/kunit/kunit-test.c index 5ebe059d16e2..555601d17f79 100644 --- a/lib/kunit/test-test.c +++ b/lib/kunit/kunit-test.c @@ -7,6 +7,8 @@ */ #include <kunit/test.h> +#include "try-catch-impl.h" + struct kunit_try_catch_test_context { struct kunit_try_catch *try_catch; bool function_called; @@ -100,7 +102,6 @@ static struct kunit_suite kunit_try_catch_test_suite = { .init = kunit_try_catch_test_init, .test_cases = kunit_try_catch_test_cases, }; -kunit_test_suite(kunit_try_catch_test_suite); /* * Context for testing test managed resources @@ -117,14 +118,14 @@ static int fake_resource_init(struct kunit_resource *res, void *context) { struct kunit_test_resource_context *ctx = context; - res->allocation = &ctx->is_resource_initialized; + res->data = &ctx->is_resource_initialized; ctx->is_resource_initialized = true; return 0; } static void fake_resource_free(struct kunit_resource *res) { - bool *is_resource_initialized = res->allocation; + bool *is_resource_initialized = res->data; *is_resource_initialized = false; } @@ -133,7 +134,7 @@ static void kunit_resource_test_init_resources(struct kunit *test) { struct kunit_test_resource_context *ctx = test->priv; - kunit_init_test(&ctx->test, "testing_test_init_test"); + kunit_init_test(&ctx->test, "testing_test_init_test", NULL); KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); } @@ -153,11 +154,21 @@ static void kunit_resource_test_alloc_resource(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, res); KUNIT_EXPECT_PTR_EQ(test, &ctx->is_resource_initialized, - (bool *) res->allocation); + (bool *)res->data); KUNIT_EXPECT_TRUE(test, list_is_last(&res->node, &ctx->test.resources)); KUNIT_EXPECT_PTR_EQ(test, free, res->free); + + kunit_put_resource(res); } +/* + * Note: tests below use kunit_alloc_and_get_resource(), so as a consequence + * they have a reference to the associated resource that they must release + * via kunit_put_resource(). In normal operation, users will only + * have to do this for cases where they use kunit_find_resource(), and the + * kunit_alloc_resource() function will be used (which does not take a + * resource reference). + */ static void kunit_resource_test_destroy_resource(struct kunit *test) { struct kunit_test_resource_context *ctx = test->priv; @@ -168,11 +179,12 @@ static void kunit_resource_test_destroy_resource(struct kunit *test) GFP_KERNEL, ctx); + kunit_put_resource(res); + KUNIT_ASSERT_FALSE(test, - kunit_resource_destroy(&ctx->test, + kunit_destroy_resource(&ctx->test, kunit_resource_instance_match, - res->free, - res->allocation)); + res->data)); KUNIT_EXPECT_FALSE(test, ctx->is_resource_initialized); KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); @@ -190,6 +202,7 @@ static void kunit_resource_test_cleanup_resources(struct kunit *test) fake_resource_free, GFP_KERNEL, ctx); + kunit_put_resource(resources[i]); } kunit_cleanup(&ctx->test); @@ -220,14 +233,14 @@ static int fake_resource_2_init(struct kunit_resource *res, void *context) KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, allocate_order, 2); - res->allocation = ctx; + res->data = ctx; return 0; } static void fake_resource_2_free(struct kunit_resource *res) { - struct kunit_test_resource_context *ctx = res->allocation; + struct kunit_test_resource_context *ctx = res->data; KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, free_order, 2); } @@ -235,23 +248,26 @@ static void fake_resource_2_free(struct kunit_resource *res) static int fake_resource_1_init(struct kunit_resource *res, void *context) { struct kunit_test_resource_context *ctx = context; + struct kunit_resource *res2; - kunit_alloc_and_get_resource(&ctx->test, - fake_resource_2_init, - fake_resource_2_free, - GFP_KERNEL, - ctx); + res2 = kunit_alloc_and_get_resource(&ctx->test, + fake_resource_2_init, + fake_resource_2_free, + GFP_KERNEL, + ctx); KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, allocate_order, 1); - res->allocation = ctx; + res->data = ctx; + + kunit_put_resource(res2); return 0; } static void fake_resource_1_free(struct kunit_resource *res) { - struct kunit_test_resource_context *ctx = res->allocation; + struct kunit_test_resource_context *ctx = res->data; KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, free_order, 1); } @@ -264,13 +280,14 @@ static void fake_resource_1_free(struct kunit_resource *res) static void kunit_resource_test_proper_free_ordering(struct kunit *test) { struct kunit_test_resource_context *ctx = test->priv; + struct kunit_resource *res; /* fake_resource_1 allocates a fake_resource_2 in its init. */ - kunit_alloc_and_get_resource(&ctx->test, - fake_resource_1_init, - fake_resource_1_free, - GFP_KERNEL, - ctx); + res = kunit_alloc_and_get_resource(&ctx->test, + fake_resource_1_init, + fake_resource_1_free, + GFP_KERNEL, + ctx); /* * Since fake_resource_2_init calls KUNIT_RESOURCE_TEST_MARK_ORDER @@ -280,6 +297,8 @@ static void kunit_resource_test_proper_free_ordering(struct kunit *test) KUNIT_EXPECT_EQ(test, ctx->allocate_order[0], 2); KUNIT_EXPECT_EQ(test, ctx->allocate_order[1], 1); + kunit_put_resource(res); + kunit_cleanup(&ctx->test); /* @@ -291,6 +310,57 @@ static void kunit_resource_test_proper_free_ordering(struct kunit *test) KUNIT_EXPECT_EQ(test, ctx->free_order[1], 2); } +static void kunit_resource_test_static(struct kunit *test) +{ + struct kunit_test_resource_context ctx; + struct kunit_resource res; + + KUNIT_EXPECT_EQ(test, kunit_add_resource(test, NULL, NULL, &res, &ctx), + 0); + + KUNIT_EXPECT_PTR_EQ(test, res.data, (void *)&ctx); + + kunit_cleanup(test); + + KUNIT_EXPECT_TRUE(test, list_empty(&test->resources)); +} + +static void kunit_resource_test_named(struct kunit *test) +{ + struct kunit_resource res1, res2, *found = NULL; + struct kunit_test_resource_context ctx; + + KUNIT_EXPECT_EQ(test, + kunit_add_named_resource(test, NULL, NULL, &res1, + "resource_1", &ctx), + 0); + KUNIT_EXPECT_PTR_EQ(test, res1.data, (void *)&ctx); + + KUNIT_EXPECT_EQ(test, + kunit_add_named_resource(test, NULL, NULL, &res1, + "resource_1", &ctx), + -EEXIST); + + KUNIT_EXPECT_EQ(test, + kunit_add_named_resource(test, NULL, NULL, &res2, + "resource_2", &ctx), + 0); + + found = kunit_find_named_resource(test, "resource_1"); + + KUNIT_EXPECT_PTR_EQ(test, found, &res1); + + if (found) + kunit_put_resource(&res1); + + KUNIT_EXPECT_EQ(test, kunit_destroy_named_resource(test, "resource_2"), + 0); + + kunit_cleanup(test); + + KUNIT_EXPECT_TRUE(test, list_empty(&test->resources)); +} + static int kunit_resource_test_init(struct kunit *test) { struct kunit_test_resource_context *ctx = @@ -300,7 +370,7 @@ static int kunit_resource_test_init(struct kunit *test) test->priv = ctx; - kunit_init_test(&ctx->test, "test_test_context"); + kunit_init_test(&ctx->test, "test_test_context", NULL); return 0; } @@ -319,6 +389,8 @@ static struct kunit_case kunit_resource_test_cases[] = { KUNIT_CASE(kunit_resource_test_destroy_resource), KUNIT_CASE(kunit_resource_test_cleanup_resources), KUNIT_CASE(kunit_resource_test_proper_free_ordering), + KUNIT_CASE(kunit_resource_test_static), + KUNIT_CASE(kunit_resource_test_named), {} }; @@ -328,4 +400,86 @@ static struct kunit_suite kunit_resource_test_suite = { .exit = kunit_resource_test_exit, .test_cases = kunit_resource_test_cases, }; -kunit_test_suite(kunit_resource_test_suite); + +static void kunit_log_test(struct kunit *test); + +static struct kunit_case kunit_log_test_cases[] = { + KUNIT_CASE(kunit_log_test), + {} +}; + +static struct kunit_suite kunit_log_test_suite = { + .name = "kunit-log-test", + .test_cases = kunit_log_test_cases, +}; + +static void kunit_log_test(struct kunit *test) +{ + struct kunit_suite suite; + + suite.log = kunit_kzalloc(test, KUNIT_LOG_SIZE, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log); + + kunit_log(KERN_INFO, test, "put this in log."); + kunit_log(KERN_INFO, test, "this too."); + kunit_log(KERN_INFO, &suite, "add to suite log."); + kunit_log(KERN_INFO, &suite, "along with this."); + +#ifdef CONFIG_KUNIT_DEBUGFS + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, + strstr(test->log, "put this in log.")); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, + strstr(test->log, "this too.")); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, + strstr(suite.log, "add to suite log.")); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, + strstr(suite.log, "along with this.")); +#else + KUNIT_EXPECT_PTR_EQ(test, test->log, (char *)NULL); +#endif +} + +static void kunit_status_set_failure_test(struct kunit *test) +{ + struct kunit fake; + + kunit_init_test(&fake, "fake test", NULL); + + KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_SUCCESS); + kunit_set_failure(&fake); + KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_FAILURE); +} + +static void kunit_status_mark_skipped_test(struct kunit *test) +{ + struct kunit fake; + + kunit_init_test(&fake, "fake test", NULL); + + /* Before: Should be SUCCESS with no comment. */ + KUNIT_EXPECT_EQ(test, fake.status, KUNIT_SUCCESS); + KUNIT_EXPECT_STREQ(test, fake.status_comment, ""); + + /* Mark the test as skipped. */ + kunit_mark_skipped(&fake, "Accepts format string: %s", "YES"); + + /* After: Should be SKIPPED with our comment. */ + KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_SKIPPED); + KUNIT_EXPECT_STREQ(test, fake.status_comment, "Accepts format string: YES"); +} + +static struct kunit_case kunit_status_test_cases[] = { + KUNIT_CASE(kunit_status_set_failure_test), + KUNIT_CASE(kunit_status_mark_skipped_test), + {} +}; + +static struct kunit_suite kunit_status_test_suite = { + .name = "kunit_status", + .test_cases = kunit_status_test_cases, +}; + +kunit_test_suites(&kunit_try_catch_test_suite, &kunit_resource_test_suite, + &kunit_log_test_suite, &kunit_status_test_suite); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c index 76cc05eb00ed..110f3a993250 100644 --- a/lib/kunit/string-stream-test.c +++ b/lib/kunit/string-stream-test.c @@ -6,10 +6,11 @@ * Author: Brendan Higgins <brendanhiggins@google.com> */ -#include <kunit/string-stream.h> #include <kunit/test.h> #include <linux/slab.h> +#include "string-stream.h" + static void string_stream_test_empty_on_creation(struct kunit *test) { struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL); @@ -49,4 +50,4 @@ static struct kunit_suite string_stream_test_suite = { .name = "string-stream-test", .test_cases = string_stream_test_cases }; -kunit_test_suite(string_stream_test_suite); +kunit_test_suites(&string_stream_test_suite); diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c index e6d17aacca30..141789ca8949 100644 --- a/lib/kunit/string-stream.c +++ b/lib/kunit/string-stream.c @@ -6,11 +6,12 @@ * Author: Brendan Higgins <brendanhiggins@google.com> */ -#include <kunit/string-stream.h> #include <kunit/test.h> #include <linux/list.h> #include <linux/slab.h> +#include "string-stream.h" + struct string_stream_fragment_alloc_context { struct kunit *test; int len; @@ -32,14 +33,14 @@ static int string_stream_fragment_init(struct kunit_resource *res, if (!frag->fragment) return -ENOMEM; - res->allocation = frag; + res->data = frag; return 0; } static void string_stream_fragment_free(struct kunit_resource *res) { - struct string_stream_fragment *frag = res->allocation; + struct string_stream_fragment *frag = res->data; list_del(&frag->node); kunit_kfree(frag->test, frag->fragment); @@ -64,9 +65,8 @@ static struct string_stream_fragment *alloc_string_stream_fragment( static int string_stream_fragment_destroy(struct string_stream_fragment *frag) { - return kunit_resource_destroy(frag->test, + return kunit_destroy_resource(frag->test, kunit_resource_instance_match, - string_stream_fragment_free, frag); } @@ -178,7 +178,7 @@ static int string_stream_init(struct kunit_resource *res, void *context) if (!stream) return -ENOMEM; - res->allocation = stream; + res->data = stream; stream->gfp = ctx->gfp; stream->test = ctx->test; INIT_LIST_HEAD(&stream->fragments); @@ -189,7 +189,7 @@ static int string_stream_init(struct kunit_resource *res, void *context) static void string_stream_free(struct kunit_resource *res) { - struct string_stream *stream = res->allocation; + struct string_stream *stream = res->data; string_stream_clear(stream); } @@ -210,8 +210,7 @@ struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp) int string_stream_destroy(struct string_stream *stream) { - return kunit_resource_destroy(stream->test, + return kunit_destroy_resource(stream->test, kunit_resource_instance_match, - string_stream_free, stream); } diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h new file mode 100644 index 000000000000..43f9508a55b4 --- /dev/null +++ b/lib/kunit/string-stream.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * C++ stream style string builder used in KUnit for building messages. + * + * Copyright (C) 2019, Google LLC. + * Author: Brendan Higgins <brendanhiggins@google.com> + */ + +#ifndef _KUNIT_STRING_STREAM_H +#define _KUNIT_STRING_STREAM_H + +#include <linux/spinlock.h> +#include <linux/types.h> +#include <linux/stdarg.h> + +struct string_stream_fragment { + struct kunit *test; + struct list_head node; + char *fragment; +}; + +struct string_stream { + size_t length; + struct list_head fragments; + /* length and fragments are protected by this lock */ + spinlock_t lock; + struct kunit *test; + gfp_t gfp; +}; + +struct kunit; + +struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp); + +int __printf(2, 3) string_stream_add(struct string_stream *stream, + const char *fmt, ...); + +int __printf(2, 0) string_stream_vadd(struct string_stream *stream, + const char *fmt, + va_list args); + +char *string_stream_get_string(struct string_stream *stream); + +int string_stream_append(struct string_stream *stream, + struct string_stream *other); + +bool string_stream_is_empty(struct string_stream *stream); + +int string_stream_destroy(struct string_stream *stream); + +#endif /* _KUNIT_STRING_STREAM_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index c83c0fa59cbd..3bd741e50a2d 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -7,92 +7,215 @@ */ #include <kunit/test.h> -#include <kunit/try-catch.h> +#include <kunit/test-bug.h> #include <linux/kernel.h> +#include <linux/kref.h> +#include <linux/moduleparam.h> #include <linux/sched/debug.h> +#include <linux/sched.h> -static void kunit_set_failure(struct kunit *test) +#include "debugfs.h" +#include "string-stream.h" +#include "try-catch-impl.h" + +#if IS_BUILTIN(CONFIG_KUNIT) +/* + * Fail the current test and print an error message to the log. + */ +void __kunit_fail_current_test(const char *file, int line, const char *fmt, ...) { - WRITE_ONCE(test->success, false); + va_list args; + int len; + char *buffer; + + if (!current->kunit_test) + return; + + kunit_set_failure(current->kunit_test); + + /* kunit_err() only accepts literals, so evaluate the args first. */ + va_start(args, fmt); + len = vsnprintf(NULL, 0, fmt, args) + 1; + va_end(args); + + buffer = kunit_kmalloc(current->kunit_test, len, GFP_KERNEL); + if (!buffer) + return; + + va_start(args, fmt); + vsnprintf(buffer, len, fmt, args); + va_end(args); + + kunit_err(current->kunit_test, "%s:%d: %s", file, line, buffer); + kunit_kfree(current->kunit_test, buffer); } +EXPORT_SYMBOL_GPL(__kunit_fail_current_test); +#endif + +/* + * KUnit statistic mode: + * 0 - disabled + * 1 - only when there is more than one subtest + * 2 - enabled + */ +static int kunit_stats_enabled = 1; +module_param_named(stats_enabled, kunit_stats_enabled, int, 0644); +MODULE_PARM_DESC(stats_enabled, + "Print test stats: never (0), only for multiple subtests (1), or always (2)"); + +struct kunit_result_stats { + unsigned long passed; + unsigned long skipped; + unsigned long failed; + unsigned long total; +}; -static void kunit_print_tap_version(void) +static bool kunit_should_print_stats(struct kunit_result_stats stats) { - static bool kunit_has_printed_tap_version; + if (kunit_stats_enabled == 0) + return false; - if (!kunit_has_printed_tap_version) { - pr_info("TAP version 14\n"); - kunit_has_printed_tap_version = true; - } + if (kunit_stats_enabled == 2) + return true; + + return (stats.total > 1); +} + +static void kunit_print_test_stats(struct kunit *test, + struct kunit_result_stats stats) +{ + if (!kunit_should_print_stats(stats)) + return; + + kunit_log(KERN_INFO, test, + KUNIT_SUBTEST_INDENT + "# %s: pass:%lu fail:%lu skip:%lu total:%lu", + test->name, + stats.passed, + stats.failed, + stats.skipped, + stats.total); } -static size_t kunit_test_cases_len(struct kunit_case *test_cases) +/* + * Append formatted message to log, size of which is limited to + * KUNIT_LOG_SIZE bytes (including null terminating byte). + */ +void kunit_log_append(char *log, const char *fmt, ...) +{ + char line[KUNIT_LOG_SIZE]; + va_list args; + int len_left; + + if (!log) + return; + + len_left = KUNIT_LOG_SIZE - strlen(log) - 1; + if (len_left <= 0) + return; + + va_start(args, fmt); + vsnprintf(line, sizeof(line), fmt, args); + va_end(args); + + strncat(log, line, len_left); +} +EXPORT_SYMBOL_GPL(kunit_log_append); + +size_t kunit_suite_num_test_cases(struct kunit_suite *suite) { struct kunit_case *test_case; size_t len = 0; - for (test_case = test_cases; test_case->run_case; test_case++) + kunit_suite_for_each_test_case(suite, test_case) len++; return len; } +EXPORT_SYMBOL_GPL(kunit_suite_num_test_cases); static void kunit_print_subtest_start(struct kunit_suite *suite) { - kunit_print_tap_version(); - pr_info("\t# Subtest: %s\n", suite->name); - pr_info("\t1..%zd\n", kunit_test_cases_len(suite->test_cases)); + kunit_log(KERN_INFO, suite, KUNIT_SUBTEST_INDENT "# Subtest: %s", + suite->name); + kunit_log(KERN_INFO, suite, KUNIT_SUBTEST_INDENT "1..%zd", + kunit_suite_num_test_cases(suite)); } -static void kunit_print_ok_not_ok(bool should_indent, - bool is_ok, +static void kunit_print_ok_not_ok(void *test_or_suite, + bool is_test, + enum kunit_status status, size_t test_number, - const char *description) + const char *description, + const char *directive) { - const char *indent, *ok_not_ok; + struct kunit_suite *suite = is_test ? NULL : test_or_suite; + struct kunit *test = is_test ? test_or_suite : NULL; + const char *directive_header = (status == KUNIT_SKIPPED) ? " # SKIP " : ""; - if (should_indent) - indent = "\t"; - else - indent = ""; - - if (is_ok) - ok_not_ok = "ok"; + /* + * We do not log the test suite results as doing so would + * mean debugfs display would consist of the test suite + * description and status prior to individual test results. + * Hence directly printk the suite status, and we will + * separately seq_printf() the suite status for the debugfs + * representation. + */ + if (suite) + pr_info("%s %zd - %s%s%s\n", + kunit_status_to_ok_not_ok(status), + test_number, description, directive_header, + (status == KUNIT_SKIPPED) ? directive : ""); else - ok_not_ok = "not ok"; - - pr_info("%s%s %zd - %s\n", indent, ok_not_ok, test_number, description); + kunit_log(KERN_INFO, test, + KUNIT_SUBTEST_INDENT "%s %zd - %s%s%s", + kunit_status_to_ok_not_ok(status), + test_number, description, directive_header, + (status == KUNIT_SKIPPED) ? directive : ""); } -static bool kunit_suite_has_succeeded(struct kunit_suite *suite) +enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite) { const struct kunit_case *test_case; + enum kunit_status status = KUNIT_SKIPPED; - for (test_case = suite->test_cases; test_case->run_case; test_case++) - if (!test_case->success) - return false; + kunit_suite_for_each_test_case(suite, test_case) { + if (test_case->status == KUNIT_FAILURE) + return KUNIT_FAILURE; + else if (test_case->status == KUNIT_SUCCESS) + status = KUNIT_SUCCESS; + } - return true; + return status; } +EXPORT_SYMBOL_GPL(kunit_suite_has_succeeded); + +static size_t kunit_suite_counter = 1; static void kunit_print_subtest_end(struct kunit_suite *suite) { - static size_t kunit_suite_counter = 1; - - kunit_print_ok_not_ok(false, + kunit_print_ok_not_ok((void *)suite, false, kunit_suite_has_succeeded(suite), kunit_suite_counter++, - suite->name); + suite->name, + suite->status_comment); } -static void kunit_print_test_case_ok_not_ok(struct kunit_case *test_case, - size_t test_number) +unsigned int kunit_test_case_num(struct kunit_suite *suite, + struct kunit_case *test_case) { - kunit_print_ok_not_ok(true, - test_case->success, - test_number, - test_case->name); + struct kunit_case *tc; + unsigned int i = 1; + + kunit_suite_for_each_test_case(suite, tc) { + if (tc == test_case) + return i; + i++; + } + + return 0; } +EXPORT_SYMBOL_GPL(kunit_test_case_num); static void kunit_print_string_stream(struct kunit *test, struct string_stream *stream) @@ -100,6 +223,9 @@ static void kunit_print_string_stream(struct kunit *test, struct string_stream_fragment *fragment; char *buf; + if (string_stream_is_empty(stream)) + return; + buf = string_stream_get_string(stream); if (!buf) { kunit_err(test, @@ -171,14 +297,20 @@ void kunit_do_assertion(struct kunit *test, if (assert->type == KUNIT_ASSERTION) kunit_abort(test); } +EXPORT_SYMBOL_GPL(kunit_do_assertion); -void kunit_init_test(struct kunit *test, const char *name) +void kunit_init_test(struct kunit *test, const char *name, char *log) { spin_lock_init(&test->lock); INIT_LIST_HEAD(&test->resources); test->name = name; - test->success = true; + test->log = log; + if (test->log) + test->log[0] = '\0'; + test->status = KUNIT_SUCCESS; + test->status_comment[0] = '\0'; } +EXPORT_SYMBOL_GPL(kunit_init_test); /* * Initializes and runs test case. Does not clean up or do post validations. @@ -232,6 +364,8 @@ static void kunit_try_run_case(void *data) struct kunit_suite *suite = ctx->suite; struct kunit_case *test_case = ctx->test_case; + current->kunit_test = test; + /* * kunit_run_case_internal may encounter a fatal error; if it does, * abort will be called, this thread will exit, and finally the parent @@ -280,177 +414,354 @@ static void kunit_catch_run_case(void *data) * occur in a test case and reports them as failures. */ static void kunit_run_case_catch_errors(struct kunit_suite *suite, - struct kunit_case *test_case) + struct kunit_case *test_case, + struct kunit *test) { struct kunit_try_catch_context context; struct kunit_try_catch *try_catch; - struct kunit test; - kunit_init_test(&test, test_case->name); - try_catch = &test.try_catch; + kunit_init_test(test, test_case->name, test_case->log); + try_catch = &test->try_catch; kunit_try_catch_init(try_catch, - &test, + test, kunit_try_run_case, kunit_catch_run_case); - context.test = &test; + context.test = test; context.suite = suite; context.test_case = test_case; kunit_try_catch_run(try_catch, &context); - test_case->success = test.success; + /* Propagate the parameter result to the test case. */ + if (test->status == KUNIT_FAILURE) + test_case->status = KUNIT_FAILURE; + else if (test_case->status != KUNIT_FAILURE && test->status == KUNIT_SUCCESS) + test_case->status = KUNIT_SUCCESS; +} + +static void kunit_print_suite_stats(struct kunit_suite *suite, + struct kunit_result_stats suite_stats, + struct kunit_result_stats param_stats) +{ + if (kunit_should_print_stats(suite_stats)) { + kunit_log(KERN_INFO, suite, + "# %s: pass:%lu fail:%lu skip:%lu total:%lu", + suite->name, + suite_stats.passed, + suite_stats.failed, + suite_stats.skipped, + suite_stats.total); + } + + if (kunit_should_print_stats(param_stats)) { + kunit_log(KERN_INFO, suite, + "# Totals: pass:%lu fail:%lu skip:%lu total:%lu", + param_stats.passed, + param_stats.failed, + param_stats.skipped, + param_stats.total); + } +} + +static void kunit_update_stats(struct kunit_result_stats *stats, + enum kunit_status status) +{ + switch (status) { + case KUNIT_SUCCESS: + stats->passed++; + break; + case KUNIT_SKIPPED: + stats->skipped++; + break; + case KUNIT_FAILURE: + stats->failed++; + break; + } + + stats->total++; +} + +static void kunit_accumulate_stats(struct kunit_result_stats *total, + struct kunit_result_stats add) +{ + total->passed += add.passed; + total->skipped += add.skipped; + total->failed += add.failed; + total->total += add.total; } int kunit_run_tests(struct kunit_suite *suite) { + char param_desc[KUNIT_PARAM_DESC_SIZE]; struct kunit_case *test_case; - size_t test_case_count = 1; + struct kunit_result_stats suite_stats = { 0 }; + struct kunit_result_stats total_stats = { 0 }; kunit_print_subtest_start(suite); - for (test_case = suite->test_cases; test_case->run_case; test_case++) { - kunit_run_case_catch_errors(suite, test_case); - kunit_print_test_case_ok_not_ok(test_case, test_case_count++); + kunit_suite_for_each_test_case(suite, test_case) { + struct kunit test = { .param_value = NULL, .param_index = 0 }; + struct kunit_result_stats param_stats = { 0 }; + test_case->status = KUNIT_SKIPPED; + + if (test_case->generate_params) { + /* Get initial param. */ + param_desc[0] = '\0'; + test.param_value = test_case->generate_params(NULL, param_desc); + } + + do { + kunit_run_case_catch_errors(suite, test_case, &test); + + if (test_case->generate_params) { + if (param_desc[0] == '\0') { + snprintf(param_desc, sizeof(param_desc), + "param-%d", test.param_index); + } + + kunit_log(KERN_INFO, &test, + KUNIT_SUBTEST_INDENT + "# %s: %s %d - %s", + test_case->name, + kunit_status_to_ok_not_ok(test.status), + test.param_index + 1, param_desc); + + /* Get next param. */ + param_desc[0] = '\0'; + test.param_value = test_case->generate_params(test.param_value, param_desc); + test.param_index++; + } + + kunit_update_stats(¶m_stats, test.status); + + } while (test.param_value); + + kunit_print_test_stats(&test, param_stats); + + kunit_print_ok_not_ok(&test, true, test_case->status, + kunit_test_case_num(suite, test_case), + test_case->name, + test.status_comment); + + kunit_update_stats(&suite_stats, test_case->status); + kunit_accumulate_stats(&total_stats, param_stats); } + kunit_print_suite_stats(suite, suite_stats, total_stats); kunit_print_subtest_end(suite); return 0; } +EXPORT_SYMBOL_GPL(kunit_run_tests); -struct kunit_resource *kunit_alloc_and_get_resource(struct kunit *test, - kunit_resource_init_t init, - kunit_resource_free_t free, - gfp_t internal_gfp, - void *context) +static void kunit_init_suite(struct kunit_suite *suite) { - struct kunit_resource *res; - int ret; + kunit_debugfs_create_suite(suite); + suite->status_comment[0] = '\0'; +} - res = kzalloc(sizeof(*res), internal_gfp); - if (!res) - return NULL; +int __kunit_test_suites_init(struct kunit_suite * const * const suites) +{ + unsigned int i; - ret = init(res, context); - if (ret) - return NULL; + for (i = 0; suites[i] != NULL; i++) { + kunit_init_suite(suites[i]); + kunit_run_tests(suites[i]); + } + return 0; +} +EXPORT_SYMBOL_GPL(__kunit_test_suites_init); + +static void kunit_exit_suite(struct kunit_suite *suite) +{ + kunit_debugfs_destroy_suite(suite); +} + +void __kunit_test_suites_exit(struct kunit_suite **suites) +{ + unsigned int i; + + for (i = 0; suites[i] != NULL; i++) + kunit_exit_suite(suites[i]); + + kunit_suite_counter = 1; +} +EXPORT_SYMBOL_GPL(__kunit_test_suites_exit); + +/* + * Used for static resources and when a kunit_resource * has been created by + * kunit_alloc_resource(). When an init function is supplied, @data is passed + * into the init function; otherwise, we simply set the resource data field to + * the data value passed in. + */ +int kunit_add_resource(struct kunit *test, + kunit_resource_init_t init, + kunit_resource_free_t free, + struct kunit_resource *res, + void *data) +{ + int ret = 0; + unsigned long flags; res->free = free; - spin_lock(&test->lock); + kref_init(&res->refcount); + + if (init) { + ret = init(res, data); + if (ret) + return ret; + } else { + res->data = data; + } + + spin_lock_irqsave(&test->lock, flags); list_add_tail(&res->node, &test->resources); - spin_unlock(&test->lock); + /* refcount for list is established by kref_init() */ + spin_unlock_irqrestore(&test->lock, flags); - return res; + return ret; } - -static void kunit_resource_free(struct kunit *test, struct kunit_resource *res) +EXPORT_SYMBOL_GPL(kunit_add_resource); + +int kunit_add_named_resource(struct kunit *test, + kunit_resource_init_t init, + kunit_resource_free_t free, + struct kunit_resource *res, + const char *name, + void *data) { - res->free(res); - kfree(res); + struct kunit_resource *existing; + + if (!name) + return -EINVAL; + + existing = kunit_find_named_resource(test, name); + if (existing) { + kunit_put_resource(existing); + return -EEXIST; + } + + res->name = name; + + return kunit_add_resource(test, init, free, res, data); } +EXPORT_SYMBOL_GPL(kunit_add_named_resource); -static struct kunit_resource *kunit_resource_find(struct kunit *test, - kunit_resource_match_t match, - kunit_resource_free_t free, - void *match_data) +struct kunit_resource *kunit_alloc_and_get_resource(struct kunit *test, + kunit_resource_init_t init, + kunit_resource_free_t free, + gfp_t internal_gfp, + void *data) { - struct kunit_resource *resource; + struct kunit_resource *res; + int ret; - lockdep_assert_held(&test->lock); + res = kzalloc(sizeof(*res), internal_gfp); + if (!res) + return NULL; - list_for_each_entry_reverse(resource, &test->resources, node) { - if (resource->free != free) - continue; - if (match(test, resource->allocation, match_data)) - return resource; + ret = kunit_add_resource(test, init, free, res, data); + if (!ret) { + /* + * bump refcount for get; kunit_resource_put() should be called + * when done. + */ + kunit_get_resource(res); + return res; } - return NULL; } +EXPORT_SYMBOL_GPL(kunit_alloc_and_get_resource); -static struct kunit_resource *kunit_resource_remove( - struct kunit *test, - kunit_resource_match_t match, - kunit_resource_free_t free, - void *match_data) +void kunit_remove_resource(struct kunit *test, struct kunit_resource *res) { - struct kunit_resource *resource; + unsigned long flags; - spin_lock(&test->lock); - resource = kunit_resource_find(test, match, free, match_data); - if (resource) - list_del(&resource->node); - spin_unlock(&test->lock); - - return resource; + spin_lock_irqsave(&test->lock, flags); + list_del(&res->node); + spin_unlock_irqrestore(&test->lock, flags); + kunit_put_resource(res); } +EXPORT_SYMBOL_GPL(kunit_remove_resource); -int kunit_resource_destroy(struct kunit *test, - kunit_resource_match_t match, - kunit_resource_free_t free, +int kunit_destroy_resource(struct kunit *test, kunit_resource_match_t match, void *match_data) { - struct kunit_resource *resource; - - resource = kunit_resource_remove(test, match, free, match_data); + struct kunit_resource *res = kunit_find_resource(test, match, + match_data); - if (!resource) + if (!res) return -ENOENT; - kunit_resource_free(test, resource); + kunit_remove_resource(test, res); + + /* We have a reference also via _find(); drop it. */ + kunit_put_resource(res); + return 0; } +EXPORT_SYMBOL_GPL(kunit_destroy_resource); -struct kunit_kmalloc_params { +struct kunit_kmalloc_array_params { + size_t n; size_t size; gfp_t gfp; }; -static int kunit_kmalloc_init(struct kunit_resource *res, void *context) +static int kunit_kmalloc_array_init(struct kunit_resource *res, void *context) { - struct kunit_kmalloc_params *params = context; + struct kunit_kmalloc_array_params *params = context; - res->allocation = kmalloc(params->size, params->gfp); - if (!res->allocation) + res->data = kmalloc_array(params->n, params->size, params->gfp); + if (!res->data) return -ENOMEM; return 0; } -static void kunit_kmalloc_free(struct kunit_resource *res) +static void kunit_kmalloc_array_free(struct kunit_resource *res) { - kfree(res->allocation); + kfree(res->data); } -void *kunit_kmalloc(struct kunit *test, size_t size, gfp_t gfp) +void *kunit_kmalloc_array(struct kunit *test, size_t n, size_t size, gfp_t gfp) { - struct kunit_kmalloc_params params = { + struct kunit_kmalloc_array_params params = { .size = size, + .n = n, .gfp = gfp }; return kunit_alloc_resource(test, - kunit_kmalloc_init, - kunit_kmalloc_free, + kunit_kmalloc_array_init, + kunit_kmalloc_array_free, gfp, ¶ms); } +EXPORT_SYMBOL_GPL(kunit_kmalloc_array); void kunit_kfree(struct kunit *test, const void *ptr) { - int rc; + struct kunit_resource *res; + + res = kunit_find_resource(test, kunit_resource_instance_match, + (void *)ptr); + + /* + * Removing the resource from the list of resources drops the + * reference count to 1; the final put will trigger the free. + */ + kunit_remove_resource(test, res); - rc = kunit_resource_destroy(test, - kunit_resource_instance_match, - kunit_kmalloc_free, - (void *)ptr); + kunit_put_resource(res); - WARN_ON(rc); } +EXPORT_SYMBOL_GPL(kunit_kfree); void kunit_cleanup(struct kunit *test) { - struct kunit_resource *resource; + struct kunit_resource *res; + unsigned long flags; /* * test->resources is a stack - each allocation must be freed in the @@ -462,17 +773,38 @@ void kunit_cleanup(struct kunit *test) * protect against the current node being deleted, not the next. */ while (true) { - spin_lock(&test->lock); + spin_lock_irqsave(&test->lock, flags); if (list_empty(&test->resources)) { - spin_unlock(&test->lock); + spin_unlock_irqrestore(&test->lock, flags); break; } - resource = list_last_entry(&test->resources, - struct kunit_resource, - node); - list_del(&resource->node); - spin_unlock(&test->lock); - - kunit_resource_free(test, resource); + res = list_last_entry(&test->resources, + struct kunit_resource, + node); + /* + * Need to unlock here as a resource may remove another + * resource, and this can't happen if the test->lock + * is held. + */ + spin_unlock_irqrestore(&test->lock, flags); + kunit_remove_resource(test, res); } + current->kunit_test = NULL; } +EXPORT_SYMBOL_GPL(kunit_cleanup); + +static int __init kunit_init(void) +{ + kunit_debugfs_init(); + + return 0; +} +late_initcall(kunit_init); + +static void __exit kunit_exit(void) +{ + kunit_debugfs_cleanup(); +} +module_exit(kunit_exit); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/kunit/try-catch-impl.h b/lib/kunit/try-catch-impl.h new file mode 100644 index 000000000000..203ba6a5e740 --- /dev/null +++ b/lib/kunit/try-catch-impl.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Internal kunit try catch implementation to be shared with tests. + * + * Copyright (C) 2019, Google LLC. + * Author: Brendan Higgins <brendanhiggins@google.com> + */ + +#ifndef _KUNIT_TRY_CATCH_IMPL_H +#define _KUNIT_TRY_CATCH_IMPL_H + +#include <kunit/try-catch.h> +#include <linux/types.h> + +struct kunit; + +static inline void kunit_try_catch_init(struct kunit_try_catch *try_catch, + struct kunit *test, + kunit_try_catch_func_t try, + kunit_try_catch_func_t catch) +{ + try_catch->test = test; + try_catch->try = try; + try_catch->catch = catch; +} + +#endif /* _KUNIT_TRY_CATCH_IMPL_H */ diff --git a/lib/kunit/try-catch.c b/lib/kunit/try-catch.c index 55686839eb61..0dd434e40487 100644 --- a/lib/kunit/try-catch.c +++ b/lib/kunit/try-catch.c @@ -8,17 +8,18 @@ */ #include <kunit/test.h> -#include <kunit/try-catch.h> #include <linux/completion.h> #include <linux/kernel.h> #include <linux/kthread.h> -#include <linux/sched/sysctl.h> + +#include "try-catch-impl.h" void __noreturn kunit_try_catch_throw(struct kunit_try_catch *try_catch) { try_catch->try_result = -EFAULT; complete_and_exit(try_catch->try_completion, -EFAULT); } +EXPORT_SYMBOL_GPL(kunit_try_catch_throw); static int kunit_generic_run_threadfn_adapter(void *data) { @@ -31,8 +32,6 @@ static int kunit_generic_run_threadfn_adapter(void *data) static unsigned long kunit_test_timeout(void) { - unsigned long timeout_msecs; - /* * TODO(brendanhiggins@google.com): We should probably have some type of * variable timeout here. The only question is what that timeout value @@ -49,22 +48,11 @@ static unsigned long kunit_test_timeout(void) * * For more background on this topic, see: * https://mike-bland.com/2011/11/01/small-medium-large.html + * + * If tests timeout due to exceeding sysctl_hung_task_timeout_secs, + * the task will be killed and an oops generated. */ - if (sysctl_hung_task_timeout_secs) { - /* - * If sysctl_hung_task is active, just set the timeout to some - * value less than that. - * - * In regards to the above TODO, if we decide on variable - * timeouts, this logic will likely need to change. - */ - timeout_msecs = (sysctl_hung_task_timeout_secs - 1) * - MSEC_PER_SEC; - } else { - timeout_msecs = 300 * MSEC_PER_SEC; /* 5 min */ - } - - return timeout_msecs; + return 300 * MSEC_PER_SEC; /* 5 min */ } void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context) @@ -106,13 +94,4 @@ void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context) try_catch->catch(try_catch->context); } - -void kunit_try_catch_init(struct kunit_try_catch *try_catch, - struct kunit *test, - kunit_try_catch_func_t try, - kunit_try_catch_func_t catch) -{ - try_catch->test = test; - try_catch->try = try; - try_catch->catch = catch; -} +EXPORT_SYMBOL_GPL(kunit_try_catch_run); diff --git a/lib/libcrc32c.c b/lib/libcrc32c.c index 77ab839644c5..5ca0d815a95d 100644 --- a/lib/libcrc32c.c +++ b/lib/libcrc32c.c @@ -12,7 +12,7 @@ * pages = {}, * month = {June}, *} - * Used by the iSCSI driver, possibly others, and derived from the + * Used by the iSCSI driver, possibly others, and derived from * the iscsi-crc.c module of the linux-iscsi driver at * http://linux-iscsi.sourceforge.net. * diff --git a/lib/linear_ranges.c b/lib/linear_ranges.c new file mode 100644 index 000000000000..a1a7dfa881de --- /dev/null +++ b/lib/linear_ranges.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * helpers to map values in a linear range to range index + * + * Original idea borrowed from regulator framework + * + * It might be useful if we could support also inversely proportional ranges? + * Copyright 2020 ROHM Semiconductors + */ + +#include <linux/errno.h> +#include <linux/export.h> +#include <linux/kernel.h> +#include <linux/linear_range.h> +#include <linux/module.h> + +/** + * linear_range_values_in_range - return the amount of values in a range + * @r: pointer to linear range where values are counted + * + * Compute the amount of values in range pointed by @r. Note, values can + * be all equal - range with selectors 0,...,2 with step 0 still contains + * 3 values even though they are all equal. + * + * Return: the amount of values in range pointed by @r + */ +unsigned int linear_range_values_in_range(const struct linear_range *r) +{ + if (!r) + return 0; + return r->max_sel - r->min_sel + 1; +} +EXPORT_SYMBOL_GPL(linear_range_values_in_range); + +/** + * linear_range_values_in_range_array - return the amount of values in ranges + * @r: pointer to array of linear ranges where values are counted + * @ranges: amount of ranges we include in computation. + * + * Compute the amount of values in ranges pointed by @r. Note, values can + * be all equal - range with selectors 0,...,2 with step 0 still contains + * 3 values even though they are all equal. + * + * Return: the amount of values in first @ranges ranges pointed by @r + */ +unsigned int linear_range_values_in_range_array(const struct linear_range *r, + int ranges) +{ + int i, values_in_range = 0; + + for (i = 0; i < ranges; i++) { + int values; + + values = linear_range_values_in_range(&r[i]); + if (!values) + return values; + + values_in_range += values; + } + return values_in_range; +} +EXPORT_SYMBOL_GPL(linear_range_values_in_range_array); + +/** + * linear_range_get_max_value - return the largest value in a range + * @r: pointer to linear range where value is looked from + * + * Return: the largest value in the given range + */ +unsigned int linear_range_get_max_value(const struct linear_range *r) +{ + return r->min + (r->max_sel - r->min_sel) * r->step; +} +EXPORT_SYMBOL_GPL(linear_range_get_max_value); + +/** + * linear_range_get_value - fetch a value from given range + * @r: pointer to linear range where value is looked from + * @selector: selector for which the value is searched + * @val: address where found value is updated + * + * Search given ranges for value which matches given selector. + * + * Return: 0 on success, -EINVAL given selector is not found from any of the + * ranges. + */ +int linear_range_get_value(const struct linear_range *r, unsigned int selector, + unsigned int *val) +{ + if (r->min_sel > selector || r->max_sel < selector) + return -EINVAL; + + *val = r->min + (selector - r->min_sel) * r->step; + + return 0; +} +EXPORT_SYMBOL_GPL(linear_range_get_value); + +/** + * linear_range_get_value_array - fetch a value from array of ranges + * @r: pointer to array of linear ranges where value is looked from + * @ranges: amount of ranges in an array + * @selector: selector for which the value is searched + * @val: address where found value is updated + * + * Search through an array of ranges for value which matches given selector. + * + * Return: 0 on success, -EINVAL given selector is not found from any of the + * ranges. + */ +int linear_range_get_value_array(const struct linear_range *r, int ranges, + unsigned int selector, unsigned int *val) +{ + int i; + + for (i = 0; i < ranges; i++) + if (r[i].min_sel <= selector && r[i].max_sel >= selector) + return linear_range_get_value(&r[i], selector, val); + + return -EINVAL; +} +EXPORT_SYMBOL_GPL(linear_range_get_value_array); + +/** + * linear_range_get_selector_low - return linear range selector for value + * @r: pointer to linear range where selector is looked from + * @val: value for which the selector is searched + * @selector: address where found selector value is updated + * @found: flag to indicate that given value was in the range + * + * Return selector for which range value is closest match for given + * input value. Value is matching if it is equal or smaller than given + * value. If given value is in the range, then @found is set true. + * + * Return: 0 on success, -EINVAL if range is invalid or does not contain + * value smaller or equal to given value + */ +int linear_range_get_selector_low(const struct linear_range *r, + unsigned int val, unsigned int *selector, + bool *found) +{ + *found = false; + + if (r->min > val) + return -EINVAL; + + if (linear_range_get_max_value(r) < val) { + *selector = r->max_sel; + return 0; + } + + *found = true; + + if (r->step == 0) + *selector = r->min_sel; + else + *selector = (val - r->min) / r->step + r->min_sel; + + return 0; +} +EXPORT_SYMBOL_GPL(linear_range_get_selector_low); + +/** + * linear_range_get_selector_low_array - return linear range selector for value + * @r: pointer to array of linear ranges where selector is looked from + * @ranges: amount of ranges to scan from array + * @val: value for which the selector is searched + * @selector: address where found selector value is updated + * @found: flag to indicate that given value was in the range + * + * Scan array of ranges for selector for which range value matches given + * input value. Value is matching if it is equal or smaller than given + * value. If given value is found to be in a range scanning is stopped and + * @found is set true. If a range with values smaller than given value is found + * but the range max is being smaller than given value, then the range's + * biggest selector is updated to @selector but scanning ranges is continued + * and @found is set to false. + * + * Return: 0 on success, -EINVAL if range array is invalid or does not contain + * range with a value smaller or equal to given value + */ +int linear_range_get_selector_low_array(const struct linear_range *r, + int ranges, unsigned int val, + unsigned int *selector, bool *found) +{ + int i; + int ret = -EINVAL; + + for (i = 0; i < ranges; i++) { + int tmpret; + + tmpret = linear_range_get_selector_low(&r[i], val, selector, + found); + if (!tmpret) + ret = 0; + + if (*found) + break; + } + + return ret; +} +EXPORT_SYMBOL_GPL(linear_range_get_selector_low_array); + +/** + * linear_range_get_selector_high - return linear range selector for value + * @r: pointer to linear range where selector is looked from + * @val: value for which the selector is searched + * @selector: address where found selector value is updated + * @found: flag to indicate that given value was in the range + * + * Return selector for which range value is closest match for given + * input value. Value is matching if it is equal or higher than given + * value. If given value is in the range, then @found is set true. + * + * Return: 0 on success, -EINVAL if range is invalid or does not contain + * value greater or equal to given value + */ +int linear_range_get_selector_high(const struct linear_range *r, + unsigned int val, unsigned int *selector, + bool *found) +{ + *found = false; + + if (linear_range_get_max_value(r) < val) + return -EINVAL; + + if (r->min > val) { + *selector = r->min_sel; + return 0; + } + + *found = true; + + if (r->step == 0) + *selector = r->max_sel; + else + *selector = DIV_ROUND_UP(val - r->min, r->step) + r->min_sel; + + return 0; +} +EXPORT_SYMBOL_GPL(linear_range_get_selector_high); + +/** + * linear_range_get_selector_within - return linear range selector for value + * @r: pointer to linear range where selector is looked from + * @val: value for which the selector is searched + * @selector: address where found selector value is updated + * + * Return selector for which range value is closest match for given + * input value. Value is matching if it is equal or lower than given + * value. But return maximum selector if given value is higher than + * maximum value. + */ +void linear_range_get_selector_within(const struct linear_range *r, + unsigned int val, unsigned int *selector) +{ + if (r->min > val) { + *selector = r->min_sel; + return; + } + + if (linear_range_get_max_value(r) < val) { + *selector = r->max_sel; + return; + } + + if (r->step == 0) + *selector = r->min_sel; + else + *selector = (val - r->min) / r->step + r->min_sel; +} +EXPORT_SYMBOL_GPL(linear_range_get_selector_within); + +MODULE_DESCRIPTION("linear-ranges helper"); +MODULE_LICENSE("GPL"); diff --git a/lib/list-test.c b/lib/list-test.c index 363c600491c3..ee09505df16f 100644 --- a/lib/list-test.c +++ b/lib/list-test.c @@ -659,7 +659,7 @@ static void list_test_list_for_each_prev_safe(struct kunit *test) static void list_test_list_for_each_entry(struct kunit *test) { struct list_test_struct entries[5], *cur; - static LIST_HEAD(list); + LIST_HEAD(list); int i = 0; for (i = 0; i < 5; ++i) { @@ -680,7 +680,7 @@ static void list_test_list_for_each_entry(struct kunit *test) static void list_test_list_for_each_entry_reverse(struct kunit *test) { struct list_test_struct entries[5], *cur; - static LIST_HEAD(list); + LIST_HEAD(list); int i = 0; for (i = 0; i < 5; ++i) { @@ -743,4 +743,6 @@ static struct kunit_suite list_test_module = { .test_cases = list_test_cases, }; -kunit_test_suite(list_test_module); +kunit_test_suites(&list_test_module); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/list_sort.c b/lib/list_sort.c index 52f0c258c895..0fb59e92ca2d 100644 --- a/lib/list_sort.c +++ b/lib/list_sort.c @@ -7,16 +7,13 @@ #include <linux/list_sort.h> #include <linux/list.h> -typedef int __attribute__((nonnull(2,3))) (*cmp_func)(void *, - struct list_head const *, struct list_head const *); - /* * Returns a list organized in an intermediate format suited * to chaining of merge() calls: null-terminated, no reserved or * sentinel head node, "prev" links not maintained. */ __attribute__((nonnull(2,3,4))) -static struct list_head *merge(void *priv, cmp_func cmp, +static struct list_head *merge(void *priv, list_cmp_func_t cmp, struct list_head *a, struct list_head *b) { struct list_head *head, **tail = &head; @@ -52,7 +49,7 @@ static struct list_head *merge(void *priv, cmp_func cmp, * throughout. */ __attribute__((nonnull(2,3,4,5))) -static void merge_final(void *priv, cmp_func cmp, struct list_head *head, +static void merge_final(void *priv, list_cmp_func_t cmp, struct list_head *head, struct list_head *a, struct list_head *b) { struct list_head *tail = head; @@ -107,7 +104,7 @@ static void merge_final(void *priv, cmp_func cmp, struct list_head *head, * @head: the list to sort * @cmp: the elements comparison function * - * The comparison funtion @cmp must return > 0 if @a should sort after + * The comparison function @cmp must return > 0 if @a should sort after * @b ("@a > @b" if you want an ascending sort), and <= 0 if @a should * sort before @b *or* their original order should be preserved. It is * always called with the element that came first in the input in @a, @@ -140,7 +137,7 @@ static void merge_final(void *priv, cmp_func cmp, struct list_head *head, * * * The merging is controlled by "count", the number of elements in the - * pending lists. This is beautiully simple code, but rather subtle. + * pending lists. This is beautifully simple code, but rather subtle. * * Each time we increment "count", we set one bit (bit k) and clear * bits k-1 .. 0. Each time this happens (except the very first time @@ -185,9 +182,7 @@ static void merge_final(void *priv, cmp_func cmp, struct list_head *head, * 2^(k+1) - 1 (second merge of case 5 when x == 2^(k-1) - 1). */ __attribute__((nonnull(2,3))) -void list_sort(void *priv, struct list_head *head, - int (*cmp)(void *priv, struct list_head *a, - struct list_head *b)) +void list_sort(void *priv, struct list_head *head, list_cmp_func_t cmp) { struct list_head *list = head->next, *pending = NULL; size_t count = 0; /* Count of pending */ @@ -227,7 +222,7 @@ void list_sort(void *priv, struct list_head *head, if (likely(bits)) { struct list_head *a = *tail, *b = a->prev; - a = merge(priv, (cmp_func)cmp, b, a); + a = merge(priv, cmp, b, a); /* Install the merged result in place of the inputs */ a->prev = b->prev; *tail = a; @@ -249,10 +244,10 @@ void list_sort(void *priv, struct list_head *head, if (!next) break; - list = merge(priv, (cmp_func)cmp, pending, list); + list = merge(priv, cmp, pending, list); pending = next; } /* The final merge, rebuilding prev links */ - merge_final(priv, (cmp_func)cmp, head, pending, list); + merge_final(priv, cmp, head, pending, list); } EXPORT_SYMBOL(list_sort); diff --git a/lib/livepatch/Makefile b/lib/livepatch/Makefile index 295b94bff370..dcc912b3478f 100644 --- a/lib/livepatch/Makefile +++ b/lib/livepatch/Makefile @@ -12,7 +12,3 @@ obj-$(CONFIG_TEST_LIVEPATCH) += test_klp_atomic_replace.o \ test_klp_state.o \ test_klp_state2.o \ test_klp_state3.o - -# Target modules to be livepatched require CC_FLAGS_FTRACE -CFLAGS_test_klp_callbacks_busy.o += $(CC_FLAGS_FTRACE) -CFLAGS_test_klp_callbacks_mod.o += $(CC_FLAGS_FTRACE) diff --git a/lib/livepatch/test_klp_callbacks_busy.c b/lib/livepatch/test_klp_callbacks_busy.c index 40beddf8a0e2..7ac845f65be5 100644 --- a/lib/livepatch/test_klp_callbacks_busy.c +++ b/lib/livepatch/test_klp_callbacks_busy.c @@ -5,34 +5,53 @@ #include <linux/module.h> #include <linux/kernel.h> +#include <linux/sched.h> #include <linux/workqueue.h> #include <linux/delay.h> -static int sleep_secs; -module_param(sleep_secs, int, 0644); -MODULE_PARM_DESC(sleep_secs, "sleep_secs (default=0)"); +/* load/run-time control from sysfs writer */ +static bool block_transition; +module_param(block_transition, bool, 0644); +MODULE_PARM_DESC(block_transition, "block_transition (default=false)"); static void busymod_work_func(struct work_struct *work); -static DECLARE_DELAYED_WORK(work, busymod_work_func); +static DECLARE_WORK(work, busymod_work_func); static void busymod_work_func(struct work_struct *work) { - pr_info("%s, sleeping %d seconds ...\n", __func__, sleep_secs); - msleep(sleep_secs * 1000); + pr_info("%s enter\n", __func__); + + while (READ_ONCE(block_transition)) { + /* + * Busy-wait until the sysfs writer has acknowledged a + * blocked transition and clears the flag. + */ + msleep(20); + } + pr_info("%s exit\n", __func__); } static int test_klp_callbacks_busy_init(void) { pr_info("%s\n", __func__); - schedule_delayed_work(&work, - msecs_to_jiffies(1000 * 0)); + schedule_work(&work); + + if (!block_transition) { + /* + * Serialize output: print all messages from the work + * function before returning from init(). + */ + flush_work(&work); + } + return 0; } static void test_klp_callbacks_busy_exit(void) { - cancel_delayed_work_sync(&work); + WRITE_ONCE(block_transition, false); + flush_work(&work); pr_info("%s\n", __func__); } diff --git a/lib/livepatch/test_klp_shadow_vars.c b/lib/livepatch/test_klp_shadow_vars.c index fe5c413efe96..b99116490858 100644 --- a/lib/livepatch/test_klp_shadow_vars.c +++ b/lib/livepatch/test_klp_shadow_vars.c @@ -60,36 +60,43 @@ static int ptr_id(void *ptr) */ static void *shadow_get(void *obj, unsigned long id) { - void *ret = klp_shadow_get(obj, id); + int **sv; + sv = klp_shadow_get(obj, id); pr_info("klp_%s(obj=PTR%d, id=0x%lx) = PTR%d\n", - __func__, ptr_id(obj), id, ptr_id(ret)); + __func__, ptr_id(obj), id, ptr_id(sv)); - return ret; + return sv; } static void *shadow_alloc(void *obj, unsigned long id, size_t size, gfp_t gfp_flags, klp_shadow_ctor_t ctor, void *ctor_data) { - void *ret = klp_shadow_alloc(obj, id, size, gfp_flags, ctor, - ctor_data); + int **var = ctor_data; + int **sv; + + sv = klp_shadow_alloc(obj, id, size, gfp_flags, ctor, var); pr_info("klp_%s(obj=PTR%d, id=0x%lx, size=%zx, gfp_flags=%pGg), ctor=PTR%d, ctor_data=PTR%d = PTR%d\n", __func__, ptr_id(obj), id, size, &gfp_flags, ptr_id(ctor), - ptr_id(ctor_data), ptr_id(ret)); - return ret; + ptr_id(*var), ptr_id(sv)); + + return sv; } static void *shadow_get_or_alloc(void *obj, unsigned long id, size_t size, gfp_t gfp_flags, klp_shadow_ctor_t ctor, void *ctor_data) { - void *ret = klp_shadow_get_or_alloc(obj, id, size, gfp_flags, ctor, - ctor_data); + int **var = ctor_data; + int **sv; + + sv = klp_shadow_get_or_alloc(obj, id, size, gfp_flags, ctor, var); pr_info("klp_%s(obj=PTR%d, id=0x%lx, size=%zx, gfp_flags=%pGg), ctor=PTR%d, ctor_data=PTR%d = PTR%d\n", __func__, ptr_id(obj), id, size, &gfp_flags, ptr_id(ctor), - ptr_id(ctor_data), ptr_id(ret)); - return ret; + ptr_id(*var), ptr_id(sv)); + + return sv; } static void shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor) @@ -102,149 +109,185 @@ static void shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor) static void shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor) { klp_shadow_free_all(id, dtor); - pr_info("klp_%s(id=0x%lx, dtor=PTR%d)\n", - __func__, id, ptr_id(dtor)); + pr_info("klp_%s(id=0x%lx, dtor=PTR%d)\n", __func__, id, ptr_id(dtor)); } /* Shadow variable constructor - remember simple pointer data */ static int shadow_ctor(void *obj, void *shadow_data, void *ctor_data) { - int **shadow_int = shadow_data; - *shadow_int = ctor_data; - pr_info("%s: PTR%d -> PTR%d\n", - __func__, ptr_id(shadow_int), ptr_id(ctor_data)); + int **sv = shadow_data; + int **var = ctor_data; + + if (!var) + return -EINVAL; + + *sv = *var; + pr_info("%s: PTR%d -> PTR%d\n", __func__, ptr_id(sv), ptr_id(*var)); return 0; } +/* + * With more than one item to free in the list, order is not determined and + * shadow_dtor will not be passed to shadow_free_all() which would make the + * test fail. (see pass 6) + */ static void shadow_dtor(void *obj, void *shadow_data) { + int **sv = shadow_data; + pr_info("%s(obj=PTR%d, shadow_data=PTR%d)\n", - __func__, ptr_id(obj), ptr_id(shadow_data)); + __func__, ptr_id(obj), ptr_id(sv)); } -static int test_klp_shadow_vars_init(void) -{ - void *obj = THIS_MODULE; - int id = 0x1234; - size_t size = sizeof(int *); - gfp_t gfp_flags = GFP_KERNEL; +/* number of objects we simulate that need shadow vars */ +#define NUM_OBJS 3 - int var1, var2, var3, var4; - int **sv1, **sv2, **sv3, **sv4; +/* dynamically created obj fields have the following shadow var id values */ +#define SV_ID1 0x1234 +#define SV_ID2 0x1235 - void *ret; +/* + * The main test case adds/removes new fields (shadow var) to each of these + * test structure instances. The last group of fields in the struct represent + * the idea that shadow variables may be added and removed to and from the + * struct during execution. + */ +struct test_object { + /* add anything here below and avoid to define an empty struct */ + struct shadow_ptr sp; + + /* these represent shadow vars added and removed with SV_ID{1,2} */ + /* char nfield1; */ + /* int nfield2; */ +}; + +static int test_klp_shadow_vars_init(void) +{ + struct test_object objs[NUM_OBJS]; + char nfields1[NUM_OBJS], *pnfields1[NUM_OBJS], **sv1[NUM_OBJS]; + char *pndup[NUM_OBJS]; + int nfields2[NUM_OBJS], *pnfields2[NUM_OBJS], **sv2[NUM_OBJS]; + void **sv; + int ret; + int i; ptr_id(NULL); - ptr_id(&var1); - ptr_id(&var2); - ptr_id(&var3); - ptr_id(&var4); /* * With an empty shadow variable hash table, expect not to find * any matches. */ - ret = shadow_get(obj, id); - if (!ret) + sv = shadow_get(&objs[0], SV_ID1); + if (!sv) pr_info(" got expected NULL result\n"); - /* - * Allocate a few shadow variables with different <obj> and <id>. - */ - sv1 = shadow_alloc(obj, id, size, gfp_flags, shadow_ctor, &var1); - if (!sv1) - return -ENOMEM; - - sv2 = shadow_alloc(obj + 1, id, size, gfp_flags, shadow_ctor, &var2); - if (!sv2) - return -ENOMEM; - - sv3 = shadow_alloc(obj, id + 1, size, gfp_flags, shadow_ctor, &var3); - if (!sv3) - return -ENOMEM; - - /* - * Verify we can find our new shadow variables and that they point - * to expected data. - */ - ret = shadow_get(obj, id); - if (!ret) - return -EINVAL; - if (ret == sv1 && *sv1 == &var1) - pr_info(" got expected PTR%d -> PTR%d result\n", - ptr_id(sv1), ptr_id(*sv1)); - - ret = shadow_get(obj + 1, id); - if (!ret) - return -EINVAL; - if (ret == sv2 && *sv2 == &var2) - pr_info(" got expected PTR%d -> PTR%d result\n", - ptr_id(sv2), ptr_id(*sv2)); - ret = shadow_get(obj, id + 1); - if (!ret) - return -EINVAL; - if (ret == sv3 && *sv3 == &var3) - pr_info(" got expected PTR%d -> PTR%d result\n", - ptr_id(sv3), ptr_id(*sv3)); - - /* - * Allocate or get a few more, this time with the same <obj>, <id>. - * The second invocation should return the same shadow var. - */ - sv4 = shadow_get_or_alloc(obj + 2, id, size, gfp_flags, shadow_ctor, &var4); - if (!sv4) - return -ENOMEM; - - ret = shadow_get_or_alloc(obj + 2, id, size, gfp_flags, shadow_ctor, &var4); - if (!ret) - return -EINVAL; - if (ret == sv4 && *sv4 == &var4) - pr_info(" got expected PTR%d -> PTR%d result\n", - ptr_id(sv4), ptr_id(*sv4)); - - /* - * Free the <obj=*, id> shadow variables and check that we can no - * longer find them. - */ - shadow_free(obj, id, shadow_dtor); /* sv1 */ - ret = shadow_get(obj, id); - if (!ret) - pr_info(" got expected NULL result\n"); + /* pass 1: init & alloc a char+int pair of svars for each objs */ + for (i = 0; i < NUM_OBJS; i++) { + pnfields1[i] = &nfields1[i]; + ptr_id(pnfields1[i]); + + if (i % 2) { + sv1[i] = shadow_alloc(&objs[i], SV_ID1, + sizeof(pnfields1[i]), GFP_KERNEL, + shadow_ctor, &pnfields1[i]); + } else { + sv1[i] = shadow_get_or_alloc(&objs[i], SV_ID1, + sizeof(pnfields1[i]), GFP_KERNEL, + shadow_ctor, &pnfields1[i]); + } + if (!sv1[i]) { + ret = -ENOMEM; + goto out; + } + + pnfields2[i] = &nfields2[i]; + ptr_id(pnfields2[i]); + sv2[i] = shadow_alloc(&objs[i], SV_ID2, sizeof(pnfields2[i]), + GFP_KERNEL, shadow_ctor, &pnfields2[i]); + if (!sv2[i]) { + ret = -ENOMEM; + goto out; + } + } - shadow_free(obj + 1, id, shadow_dtor); /* sv2 */ - ret = shadow_get(obj + 1, id); - if (!ret) - pr_info(" got expected NULL result\n"); + /* pass 2: verify we find allocated svars and where they point to */ + for (i = 0; i < NUM_OBJS; i++) { + /* check the "char" svar for all objects */ + sv = shadow_get(&objs[i], SV_ID1); + if (!sv) { + ret = -EINVAL; + goto out; + } + if ((char **)sv == sv1[i] && *sv1[i] == pnfields1[i]) + pr_info(" got expected PTR%d -> PTR%d result\n", + ptr_id(sv1[i]), ptr_id(*sv1[i])); + + /* check the "int" svar for all objects */ + sv = shadow_get(&objs[i], SV_ID2); + if (!sv) { + ret = -EINVAL; + goto out; + } + if ((int **)sv == sv2[i] && *sv2[i] == pnfields2[i]) + pr_info(" got expected PTR%d -> PTR%d result\n", + ptr_id(sv2[i]), ptr_id(*sv2[i])); + } - shadow_free(obj + 2, id, shadow_dtor); /* sv4 */ - ret = shadow_get(obj + 2, id); - if (!ret) - pr_info(" got expected NULL result\n"); + /* pass 3: verify that 'get_or_alloc' returns already allocated svars */ + for (i = 0; i < NUM_OBJS; i++) { + pndup[i] = &nfields1[i]; + ptr_id(pndup[i]); + + sv = shadow_get_or_alloc(&objs[i], SV_ID1, sizeof(pndup[i]), + GFP_KERNEL, shadow_ctor, &pndup[i]); + if (!sv) { + ret = -EINVAL; + goto out; + } + if ((char **)sv == sv1[i] && *sv1[i] == pnfields1[i]) + pr_info(" got expected PTR%d -> PTR%d result\n", + ptr_id(sv1[i]), ptr_id(*sv1[i])); + } - /* - * We should still find an <id+1> variable. - */ - ret = shadow_get(obj, id + 1); - if (!ret) - return -EINVAL; - if (ret == sv3 && *sv3 == &var3) - pr_info(" got expected PTR%d -> PTR%d result\n", - ptr_id(sv3), ptr_id(*sv3)); + /* pass 4: free <objs[*], SV_ID1> pairs of svars, verify removal */ + for (i = 0; i < NUM_OBJS; i++) { + shadow_free(&objs[i], SV_ID1, shadow_dtor); /* 'char' pairs */ + sv = shadow_get(&objs[i], SV_ID1); + if (!sv) + pr_info(" got expected NULL result\n"); + } - /* - * Free all the <id+1> variables, too. - */ - shadow_free_all(id + 1, shadow_dtor); /* sv3 */ - ret = shadow_get(obj, id); - if (!ret) - pr_info(" shadow_get() got expected NULL result\n"); + /* pass 5: check we still find <objs[*], SV_ID2> svar pairs */ + for (i = 0; i < NUM_OBJS; i++) { + sv = shadow_get(&objs[i], SV_ID2); /* 'int' pairs */ + if (!sv) { + ret = -EINVAL; + goto out; + } + if ((int **)sv == sv2[i] && *sv2[i] == pnfields2[i]) + pr_info(" got expected PTR%d -> PTR%d result\n", + ptr_id(sv2[i]), ptr_id(*sv2[i])); + } + /* pass 6: free all the <objs[*], SV_ID2> svar pairs too. */ + shadow_free_all(SV_ID2, NULL); /* 'int' pairs */ + for (i = 0; i < NUM_OBJS; i++) { + sv = shadow_get(&objs[i], SV_ID2); + if (!sv) + pr_info(" got expected NULL result\n"); + } free_ptr_list(); return 0; +out: + shadow_free_all(SV_ID1, NULL); /* 'char' pairs */ + shadow_free_all(SV_ID2, NULL); /* 'int' pairs */ + free_ptr_list(); + + return ret; } static void test_klp_shadow_vars_exit(void) diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index 14f44f59e733..71652e1c397c 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -15,6 +15,7 @@ #include <linux/mutex.h> #include <linux/ww_mutex.h> #include <linux/sched.h> +#include <linux/sched/mm.h> #include <linux/delay.h> #include <linux/lockdep.h> #include <linux/spinlock.h> @@ -23,11 +24,13 @@ #include <linux/debug_locks.h> #include <linux/irqflags.h> #include <linux/rtmutex.h> +#include <linux/local_lock.h> /* * Change this to 1 if you want to see the failure printouts: */ static unsigned int debug_locks_verbose; +unsigned int force_read_lock_recursive; static DEFINE_WD_CLASS(ww_lockdep); @@ -49,6 +52,8 @@ __setup("debug_locks_verbose=", setup_debug_locks_verbose); #define LOCKTYPE_RWSEM 0x8 #define LOCKTYPE_WW 0x10 #define LOCKTYPE_RTMUTEX 0x20 +#define LOCKTYPE_LL 0x40 +#define LOCKTYPE_SPECIAL 0x80 static struct ww_acquire_ctx t, t2; static struct ww_mutex o, o2, o3; @@ -57,10 +62,13 @@ static struct ww_mutex o, o2, o3; * Normal standalone locks, for the circular and irq-context * dependency tests: */ -static DEFINE_RAW_SPINLOCK(lock_A); -static DEFINE_RAW_SPINLOCK(lock_B); -static DEFINE_RAW_SPINLOCK(lock_C); -static DEFINE_RAW_SPINLOCK(lock_D); +static DEFINE_SPINLOCK(lock_A); +static DEFINE_SPINLOCK(lock_B); +static DEFINE_SPINLOCK(lock_C); +static DEFINE_SPINLOCK(lock_D); + +static DEFINE_RAW_SPINLOCK(raw_lock_A); +static DEFINE_RAW_SPINLOCK(raw_lock_B); static DEFINE_RWLOCK(rwlock_A); static DEFINE_RWLOCK(rwlock_B); @@ -92,12 +100,12 @@ static DEFINE_RT_MUTEX(rtmutex_D); * but X* and Y* are different classes. We do this so that * we do not trigger a real lockup: */ -static DEFINE_RAW_SPINLOCK(lock_X1); -static DEFINE_RAW_SPINLOCK(lock_X2); -static DEFINE_RAW_SPINLOCK(lock_Y1); -static DEFINE_RAW_SPINLOCK(lock_Y2); -static DEFINE_RAW_SPINLOCK(lock_Z1); -static DEFINE_RAW_SPINLOCK(lock_Z2); +static DEFINE_SPINLOCK(lock_X1); +static DEFINE_SPINLOCK(lock_X2); +static DEFINE_SPINLOCK(lock_Y1); +static DEFINE_SPINLOCK(lock_Y2); +static DEFINE_SPINLOCK(lock_Z1); +static DEFINE_SPINLOCK(lock_Z2); static DEFINE_RWLOCK(rwlock_X1); static DEFINE_RWLOCK(rwlock_X2); @@ -131,16 +139,18 @@ static DEFINE_RT_MUTEX(rtmutex_Z2); #endif +static local_lock_t local_A = INIT_LOCAL_LOCK(local_A); + /* * non-inlined runtime initializers, to let separate locks share * the same lock-class: */ #define INIT_CLASS_FUNC(class) \ static noinline void \ -init_class_##class(raw_spinlock_t *lock, rwlock_t *rwlock, \ +init_class_##class(spinlock_t *lock, rwlock_t *rwlock, \ struct mutex *mutex, struct rw_semaphore *rwsem)\ { \ - raw_spin_lock_init(lock); \ + spin_lock_init(lock); \ rwlock_init(rwlock); \ mutex_init(mutex); \ init_rwsem(rwsem); \ @@ -185,6 +195,7 @@ static void init_shared_classes(void) #define HARDIRQ_ENTER() \ local_irq_disable(); \ __irq_enter(); \ + lockdep_hardirq_threaded(); \ WARN_ON(!in_irq()); #define HARDIRQ_EXIT() \ @@ -209,10 +220,10 @@ static void init_shared_classes(void) * Shortcuts for lock/unlock API variants, to keep * the testcases compact: */ -#define L(x) raw_spin_lock(&lock_##x) -#define U(x) raw_spin_unlock(&lock_##x) +#define L(x) spin_lock(&lock_##x) +#define U(x) spin_unlock(&lock_##x) #define LU(x) L(x); U(x) -#define SI(x) raw_spin_lock_init(&lock_##x) +#define SI(x) spin_lock_init(&lock_##x) #define WL(x) write_lock(&rwlock_##x) #define WU(x) write_unlock(&rwlock_##x) @@ -247,7 +258,7 @@ static void init_shared_classes(void) #define WWAF(x) ww_acquire_fini(x) #define WWL(x, c) ww_mutex_lock(x, c) -#define WWT(x) ww_mutex_trylock(x) +#define WWT(x) ww_mutex_trylock(x, NULL) #define WWL1(x) ww_mutex_lock(x, NULL) #define WWU(x) ww_mutex_unlock(x) @@ -399,6 +410,49 @@ static void rwsem_ABBA1(void) * read_lock(A) * spin_lock(B) * spin_lock(B) + * write_lock(A) + * + * This test case is aimed at poking whether the chain cache prevents us from + * detecting a read-lock/lock-write deadlock: if the chain cache doesn't differ + * read/write locks, the following case may happen + * + * { read_lock(A)->lock(B) dependency exists } + * + * P0: + * lock(B); + * read_lock(A); + * + * { Not a deadlock, B -> A is added in the chain cache } + * + * P1: + * lock(B); + * write_lock(A); + * + * { B->A found in chain cache, not reported as a deadlock } + * + */ +static void rlock_chaincache_ABBA1(void) +{ + RL(X1); + L(Y1); + U(Y1); + RU(X1); + + L(Y1); + RL(X1); + RU(X1); + U(Y1); + + L(Y1); + WL(X1); + WU(X1); + U(Y1); // should fail +} + +/* + * read_lock(A) + * spin_lock(B) + * spin_lock(B) * read_lock(A) */ static void rlock_ABBA2(void) @@ -991,6 +1045,133 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_wlock) #undef E3 /* + * write-read / write-read / write-read deadlock even if read is recursive + */ + +#define E1() \ + \ + WL(X1); \ + RL(Y1); \ + RU(Y1); \ + WU(X1); + +#define E2() \ + \ + WL(Y1); \ + RL(Z1); \ + RU(Z1); \ + WU(Y1); + +#define E3() \ + \ + WL(Z1); \ + RL(X1); \ + RU(X1); \ + WU(Z1); + +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(W1R2_W2R3_W3R1) + +#undef E1 +#undef E2 +#undef E3 + +/* + * write-write / read-read / write-read deadlock even if read is recursive + */ + +#define E1() \ + \ + WL(X1); \ + WL(Y1); \ + WU(Y1); \ + WU(X1); + +#define E2() \ + \ + RL(Y1); \ + RL(Z1); \ + RU(Z1); \ + RU(Y1); + +#define E3() \ + \ + WL(Z1); \ + RL(X1); \ + RU(X1); \ + WU(Z1); + +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_W3R1) + +#undef E1 +#undef E2 +#undef E3 + +/* + * write-write / read-read / read-write is not deadlock when read is recursive + */ + +#define E1() \ + \ + WL(X1); \ + WL(Y1); \ + WU(Y1); \ + WU(X1); + +#define E2() \ + \ + RL(Y1); \ + RL(Z1); \ + RU(Z1); \ + RU(Y1); + +#define E3() \ + \ + RL(Z1); \ + WL(X1); \ + WU(X1); \ + RU(Z1); + +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(W1R2_R2R3_W3W1) + +#undef E1 +#undef E2 +#undef E3 + +/* + * write-read / read-read / write-write is not deadlock when read is recursive + */ + +#define E1() \ + \ + WL(X1); \ + RL(Y1); \ + RU(Y1); \ + WU(X1); + +#define E2() \ + \ + RL(Y1); \ + RL(Z1); \ + RU(Z1); \ + RU(Y1); + +#define E3() \ + \ + WL(Z1); \ + WL(X1); \ + WU(X1); \ + WU(Z1); + +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_R3W1) + +#undef E1 +#undef E2 +#undef E3 +/* * read-lock / write-lock recursion that is actually safe. */ @@ -1009,20 +1190,28 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_wlock) #define E3() \ \ IRQ_ENTER(); \ - RL(A); \ + LOCK(A); \ L(B); \ U(B); \ - RU(A); \ + UNLOCK(A); \ IRQ_EXIT(); /* - * Generate 12 testcases: + * Generate 24 testcases: */ #include "locking-selftest-hardirq.h" -GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard) +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_rlock) + +#include "locking-selftest-wlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_wlock) #include "locking-selftest-softirq.h" -GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft) +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_rlock) + +#include "locking-selftest-wlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_wlock) #undef E1 #undef E2 @@ -1036,8 +1225,8 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft) \ IRQ_DISABLE(); \ L(B); \ - WL(A); \ - WU(A); \ + LOCK(A); \ + UNLOCK(A); \ U(B); \ IRQ_ENABLE(); @@ -1054,29 +1243,95 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft) IRQ_EXIT(); /* - * Generate 12 testcases: + * Generate 24 testcases: + */ +#include "locking-selftest-hardirq.h" +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_rlock) + +#include "locking-selftest-wlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_wlock) + +#include "locking-selftest-softirq.h" +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_rlock) + +#include "locking-selftest-wlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_wlock) + +#undef E1 +#undef E2 +#undef E3 +/* + * read-lock / write-lock recursion that is unsafe. + * + * A is a ENABLED_*_READ lock + * B is a USED_IN_*_READ lock + * + * read_lock(A); + * write_lock(B); + * <interrupt> + * read_lock(B); + * write_lock(A); // if this one is read_lock(), no deadlock + */ + +#define E1() \ + \ + IRQ_DISABLE(); \ + WL(B); \ + LOCK(A); \ + UNLOCK(A); \ + WU(B); \ + IRQ_ENABLE(); + +#define E2() \ + \ + RL(A); \ + RU(A); \ + +#define E3() \ + \ + IRQ_ENTER(); \ + RL(B); \ + RU(B); \ + IRQ_EXIT(); + +/* + * Generate 24 testcases: */ #include "locking-selftest-hardirq.h" -// GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard) +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_rlock) + +#include "locking-selftest-wlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_wlock) #include "locking-selftest-softirq.h" -// GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft) +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_rlock) + +#include "locking-selftest-wlock.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_wlock) #ifdef CONFIG_DEBUG_LOCK_ALLOC # define I_SPINLOCK(x) lockdep_reset_lock(&lock_##x.dep_map) +# define I_RAW_SPINLOCK(x) lockdep_reset_lock(&raw_lock_##x.dep_map) # define I_RWLOCK(x) lockdep_reset_lock(&rwlock_##x.dep_map) # define I_MUTEX(x) lockdep_reset_lock(&mutex_##x.dep_map) # define I_RWSEM(x) lockdep_reset_lock(&rwsem_##x.dep_map) # define I_WW(x) lockdep_reset_lock(&x.dep_map) +# define I_LOCAL_LOCK(x) lockdep_reset_lock(&local_##x.dep_map) #ifdef CONFIG_RT_MUTEXES # define I_RTMUTEX(x) lockdep_reset_lock(&rtmutex_##x.dep_map) #endif #else # define I_SPINLOCK(x) +# define I_RAW_SPINLOCK(x) # define I_RWLOCK(x) # define I_MUTEX(x) # define I_RWSEM(x) # define I_WW(x) +# define I_LOCAL_LOCK(x) #endif #ifndef I_RTMUTEX @@ -1100,7 +1355,7 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft) #define I2(x) \ do { \ - raw_spin_lock_init(&lock_##x); \ + spin_lock_init(&lock_##x); \ rwlock_init(&rwlock_##x); \ mutex_init(&mutex_##x); \ init_rwsem(&rwsem_##x); \ @@ -1116,9 +1371,16 @@ static void reset_locks(void) I1(A); I1(B); I1(C); I1(D); I1(X1); I1(X2); I1(Y1); I1(Y2); I1(Z1); I1(Z2); I_WW(t); I_WW(t2); I_WW(o.base); I_WW(o2.base); I_WW(o3.base); + I_RAW_SPINLOCK(A); I_RAW_SPINLOCK(B); + I_LOCAL_LOCK(A); + lockdep_reset(); + I2(A); I2(B); I2(C); I2(D); init_shared_classes(); + raw_spin_lock_init(&raw_lock_A); + raw_spin_lock_init(&raw_lock_B); + local_lock_init(&local_A); ww_mutex_init(&o, &ww_lockdep); ww_mutex_init(&o2, &ww_lockdep); ww_mutex_init(&o3, &ww_lockdep); memset(&t, 0, sizeof(t)); memset(&t2, 0, sizeof(t2)); @@ -1140,6 +1402,8 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) WARN_ON(irqs_disabled()); + debug_locks_silent = !(debug_locks_verbose & lockclass_mask); + testcase_fn(); /* * Filter out expected failures: @@ -1160,7 +1424,7 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) } testcase_total++; - if (debug_locks_verbose) + if (debug_locks_verbose & lockclass_mask) pr_cont(" lockclass mask: %x, debug_locks: %d, expected: %d\n", lockclass_mask, debug_locks, expected); /* @@ -1199,6 +1463,19 @@ static inline void print_testname(const char *testname) dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK); \ pr_cont("\n"); +#define DO_TESTCASE_1RR(desc, name, nr) \ + print_testname(desc"/"#nr); \ + pr_cont(" |"); \ + dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK); \ + pr_cont("\n"); + +#define DO_TESTCASE_1RRB(desc, name, nr) \ + print_testname(desc"/"#nr); \ + pr_cont(" |"); \ + dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK); \ + pr_cont("\n"); + + #define DO_TESTCASE_3(desc, name, nr) \ print_testname(desc"/"#nr); \ dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN); \ @@ -1213,6 +1490,25 @@ static inline void print_testname(const char *testname) dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK); \ pr_cont("\n"); +#define DO_TESTCASE_2RW(desc, name, nr) \ + print_testname(desc"/"#nr); \ + pr_cont(" |"); \ + dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK); \ + dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK); \ + pr_cont("\n"); + +#define DO_TESTCASE_2x2RW(desc, name, nr) \ + DO_TESTCASE_2RW("hard-"desc, name##_hard, nr) \ + DO_TESTCASE_2RW("soft-"desc, name##_soft, nr) \ + +#define DO_TESTCASE_6x2x2RW(desc, name) \ + DO_TESTCASE_2x2RW(desc, name, 123); \ + DO_TESTCASE_2x2RW(desc, name, 132); \ + DO_TESTCASE_2x2RW(desc, name, 213); \ + DO_TESTCASE_2x2RW(desc, name, 231); \ + DO_TESTCASE_2x2RW(desc, name, 312); \ + DO_TESTCASE_2x2RW(desc, name, 321); + #define DO_TESTCASE_6(desc, name) \ print_testname(desc); \ dotest(name##_spin, FAILURE, LOCKTYPE_SPIN); \ @@ -1289,6 +1585,22 @@ static inline void print_testname(const char *testname) DO_TESTCASE_2IB(desc, name, 312); \ DO_TESTCASE_2IB(desc, name, 321); +#define DO_TESTCASE_6x1RR(desc, name) \ + DO_TESTCASE_1RR(desc, name, 123); \ + DO_TESTCASE_1RR(desc, name, 132); \ + DO_TESTCASE_1RR(desc, name, 213); \ + DO_TESTCASE_1RR(desc, name, 231); \ + DO_TESTCASE_1RR(desc, name, 312); \ + DO_TESTCASE_1RR(desc, name, 321); + +#define DO_TESTCASE_6x1RRB(desc, name) \ + DO_TESTCASE_1RRB(desc, name, 123); \ + DO_TESTCASE_1RRB(desc, name, 132); \ + DO_TESTCASE_1RRB(desc, name, 213); \ + DO_TESTCASE_1RRB(desc, name, 231); \ + DO_TESTCASE_1RRB(desc, name, 312); \ + DO_TESTCASE_1RRB(desc, name, 321); + #define DO_TESTCASE_6x6(desc, name) \ DO_TESTCASE_6I(desc, name, 123); \ DO_TESTCASE_6I(desc, name, 132); \ @@ -1716,10 +2028,23 @@ static void ww_test_edeadlk_acquire_wrong_slow(void) static void ww_test_spin_nest_unlocked(void) { - raw_spin_lock_nest_lock(&lock_A, &o.base); + spin_lock_nest_lock(&lock_A, &o.base); U(A); } +/* This is not a deadlock, because we have X1 to serialize Y1 and Y2 */ +static void ww_test_spin_nest_lock(void) +{ + spin_lock(&lock_X1); + spin_lock_nest_lock(&lock_Y1, &lock_X1); + spin_lock(&lock_A); + spin_lock_nest_lock(&lock_Y2, &lock_X1); + spin_unlock(&lock_A); + spin_unlock(&lock_Y2); + spin_unlock(&lock_Y1); + spin_unlock(&lock_X1); +} + static void ww_test_unneeded_slow(void) { WWAI(&t); @@ -1937,6 +2262,10 @@ static void ww_tests(void) dotest(ww_test_spin_nest_unlocked, FAILURE, LOCKTYPE_WW); pr_cont("\n"); + print_testname("spinlock nest test"); + dotest(ww_test_spin_nest_lock, SUCCESS, LOCKTYPE_WW); + pr_cont("\n"); + printk(" -----------------------------------------------------\n"); printk(" |block | try |context|\n"); printk(" -----------------------------------------------------\n"); @@ -1966,6 +2295,500 @@ static void ww_tests(void) pr_cont("\n"); } + +/* + * <in hardirq handler> + * read_lock(&A); + * <hardirq disable> + * spin_lock(&B); + * spin_lock(&B); + * read_lock(&A); + * + * is a deadlock. + */ +static void queued_read_lock_hardirq_RE_Er(void) +{ + HARDIRQ_ENTER(); + read_lock(&rwlock_A); + LOCK(B); + UNLOCK(B); + read_unlock(&rwlock_A); + HARDIRQ_EXIT(); + + HARDIRQ_DISABLE(); + LOCK(B); + read_lock(&rwlock_A); + read_unlock(&rwlock_A); + UNLOCK(B); + HARDIRQ_ENABLE(); +} + +/* + * <in hardirq handler> + * spin_lock(&B); + * <hardirq disable> + * read_lock(&A); + * read_lock(&A); + * spin_lock(&B); + * + * is not a deadlock. + */ +static void queued_read_lock_hardirq_ER_rE(void) +{ + HARDIRQ_ENTER(); + LOCK(B); + read_lock(&rwlock_A); + read_unlock(&rwlock_A); + UNLOCK(B); + HARDIRQ_EXIT(); + + HARDIRQ_DISABLE(); + read_lock(&rwlock_A); + LOCK(B); + UNLOCK(B); + read_unlock(&rwlock_A); + HARDIRQ_ENABLE(); +} + +/* + * <hardirq disable> + * spin_lock(&B); + * read_lock(&A); + * <in hardirq handler> + * spin_lock(&B); + * read_lock(&A); + * + * is a deadlock. Because the two read_lock()s are both non-recursive readers. + */ +static void queued_read_lock_hardirq_inversion(void) +{ + + HARDIRQ_ENTER(); + LOCK(B); + UNLOCK(B); + HARDIRQ_EXIT(); + + HARDIRQ_DISABLE(); + LOCK(B); + read_lock(&rwlock_A); + read_unlock(&rwlock_A); + UNLOCK(B); + HARDIRQ_ENABLE(); + + read_lock(&rwlock_A); + read_unlock(&rwlock_A); +} + +static void queued_read_lock_tests(void) +{ + printk(" --------------------------------------------------------------------------\n"); + printk(" | queued read lock tests |\n"); + printk(" ---------------------------\n"); + print_testname("hardirq read-lock/lock-read"); + dotest(queued_read_lock_hardirq_RE_Er, FAILURE, LOCKTYPE_RWLOCK); + pr_cont("\n"); + + print_testname("hardirq lock-read/read-lock"); + dotest(queued_read_lock_hardirq_ER_rE, SUCCESS, LOCKTYPE_RWLOCK); + pr_cont("\n"); + + print_testname("hardirq inversion"); + dotest(queued_read_lock_hardirq_inversion, FAILURE, LOCKTYPE_RWLOCK); + pr_cont("\n"); +} + +static void fs_reclaim_correct_nesting(void) +{ + fs_reclaim_acquire(GFP_KERNEL); + might_alloc(GFP_NOFS); + fs_reclaim_release(GFP_KERNEL); +} + +static void fs_reclaim_wrong_nesting(void) +{ + fs_reclaim_acquire(GFP_KERNEL); + might_alloc(GFP_KERNEL); + fs_reclaim_release(GFP_KERNEL); +} + +static void fs_reclaim_protected_nesting(void) +{ + unsigned int flags; + + fs_reclaim_acquire(GFP_KERNEL); + flags = memalloc_nofs_save(); + might_alloc(GFP_KERNEL); + memalloc_nofs_restore(flags); + fs_reclaim_release(GFP_KERNEL); +} + +static void fs_reclaim_tests(void) +{ + printk(" --------------------\n"); + printk(" | fs_reclaim tests |\n"); + printk(" --------------------\n"); + + print_testname("correct nesting"); + dotest(fs_reclaim_correct_nesting, SUCCESS, 0); + pr_cont("\n"); + + print_testname("wrong nesting"); + dotest(fs_reclaim_wrong_nesting, FAILURE, 0); + pr_cont("\n"); + + print_testname("protected nesting"); + dotest(fs_reclaim_protected_nesting, SUCCESS, 0); + pr_cont("\n"); +} + +#define __guard(cleanup) __maybe_unused __attribute__((__cleanup__(cleanup))) + +static void hardirq_exit(int *_) +{ + HARDIRQ_EXIT(); +} + +#define HARDIRQ_CONTEXT(name, ...) \ + int hardirq_guard_##name __guard(hardirq_exit); \ + HARDIRQ_ENTER(); + +#define NOTTHREADED_HARDIRQ_CONTEXT(name, ...) \ + int notthreaded_hardirq_guard_##name __guard(hardirq_exit); \ + local_irq_disable(); \ + __irq_enter(); \ + WARN_ON(!in_irq()); + +static void softirq_exit(int *_) +{ + SOFTIRQ_EXIT(); +} + +#define SOFTIRQ_CONTEXT(name, ...) \ + int softirq_guard_##name __guard(softirq_exit); \ + SOFTIRQ_ENTER(); + +static void rcu_exit(int *_) +{ + rcu_read_unlock(); +} + +#define RCU_CONTEXT(name, ...) \ + int rcu_guard_##name __guard(rcu_exit); \ + rcu_read_lock(); + +static void rcu_bh_exit(int *_) +{ + rcu_read_unlock_bh(); +} + +#define RCU_BH_CONTEXT(name, ...) \ + int rcu_bh_guard_##name __guard(rcu_bh_exit); \ + rcu_read_lock_bh(); + +static void rcu_sched_exit(int *_) +{ + rcu_read_unlock_sched(); +} + +#define RCU_SCHED_CONTEXT(name, ...) \ + int rcu_sched_guard_##name __guard(rcu_sched_exit); \ + rcu_read_lock_sched(); + +static void raw_spinlock_exit(raw_spinlock_t **lock) +{ + raw_spin_unlock(*lock); +} + +#define RAW_SPINLOCK_CONTEXT(name, lock) \ + raw_spinlock_t *raw_spinlock_guard_##name __guard(raw_spinlock_exit) = &(lock); \ + raw_spin_lock(&(lock)); + +static void spinlock_exit(spinlock_t **lock) +{ + spin_unlock(*lock); +} + +#define SPINLOCK_CONTEXT(name, lock) \ + spinlock_t *spinlock_guard_##name __guard(spinlock_exit) = &(lock); \ + spin_lock(&(lock)); + +static void mutex_exit(struct mutex **lock) +{ + mutex_unlock(*lock); +} + +#define MUTEX_CONTEXT(name, lock) \ + struct mutex *mutex_guard_##name __guard(mutex_exit) = &(lock); \ + mutex_lock(&(lock)); + +#define GENERATE_2_CONTEXT_TESTCASE(outer, outer_lock, inner, inner_lock) \ + \ +static void __maybe_unused inner##_in_##outer(void) \ +{ \ + outer##_CONTEXT(_, outer_lock); \ + { \ + inner##_CONTEXT(_, inner_lock); \ + } \ +} + +/* + * wait contexts (considering PREEMPT_RT) + * + * o: inner is allowed in outer + * x: inner is disallowed in outer + * + * \ inner | RCU | RAW_SPIN | SPIN | MUTEX + * outer \ | | | | + * ---------------+-------+----------+------+------- + * HARDIRQ | o | o | o | x + * ---------------+-------+----------+------+------- + * NOTTHREADED_IRQ| o | o | x | x + * ---------------+-------+----------+------+------- + * SOFTIRQ | o | o | o | x + * ---------------+-------+----------+------+------- + * RCU | o | o | o | x + * ---------------+-------+----------+------+------- + * RCU_BH | o | o | o | x + * ---------------+-------+----------+------+------- + * RCU_SCHED | o | o | x | x + * ---------------+-------+----------+------+------- + * RAW_SPIN | o | o | x | x + * ---------------+-------+----------+------+------- + * SPIN | o | o | o | x + * ---------------+-------+----------+------+------- + * MUTEX | o | o | o | o + * ---------------+-------+----------+------+------- + */ + +#define GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(HARDIRQ, , inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(NOTTHREADED_HARDIRQ, , inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(SOFTIRQ, , inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(RCU, , inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(RCU_BH, , inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(RCU_SCHED, , inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(RAW_SPINLOCK, raw_lock_A, inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(SPINLOCK, lock_A, inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(MUTEX, mutex_A, inner, inner_lock) + +GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(RCU, ) +GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(RAW_SPINLOCK, raw_lock_B) +GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(SPINLOCK, lock_B) +GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(MUTEX, mutex_B) + +/* the outer context allows all kinds of preemption */ +#define DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(outer) \ + dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK); \ + dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ + dotest(SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ + dotest(MUTEX_in_##outer, SUCCESS, LOCKTYPE_MUTEX); \ + +/* + * the outer context only allows the preemption introduced by spinlock_t (which + * is a sleepable lock for PREEMPT_RT) + */ +#define DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(outer) \ + dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK); \ + dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ + dotest(SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ + dotest(MUTEX_in_##outer, FAILURE, LOCKTYPE_MUTEX); \ + +/* the outer doesn't allows any kind of preemption */ +#define DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(outer) \ + dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK); \ + dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ + dotest(SPINLOCK_in_##outer, FAILURE, LOCKTYPE_SPIN); \ + dotest(MUTEX_in_##outer, FAILURE, LOCKTYPE_MUTEX); \ + +static void wait_context_tests(void) +{ + printk(" --------------------------------------------------------------------------\n"); + printk(" | wait context tests |\n"); + printk(" --------------------------------------------------------------------------\n"); + printk(" | rcu | raw | spin |mutex |\n"); + printk(" --------------------------------------------------------------------------\n"); + print_testname("in hardirq context"); + DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(HARDIRQ); + pr_cont("\n"); + + print_testname("in hardirq context (not threaded)"); + DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(NOTTHREADED_HARDIRQ); + pr_cont("\n"); + + print_testname("in softirq context"); + DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(SOFTIRQ); + pr_cont("\n"); + + print_testname("in RCU context"); + DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU); + pr_cont("\n"); + + print_testname("in RCU-bh context"); + DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU_BH); + pr_cont("\n"); + + print_testname("in RCU-sched context"); + DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(RCU_SCHED); + pr_cont("\n"); + + print_testname("in RAW_SPINLOCK context"); + DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(RAW_SPINLOCK); + pr_cont("\n"); + + print_testname("in SPINLOCK context"); + DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(SPINLOCK); + pr_cont("\n"); + + print_testname("in MUTEX context"); + DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(MUTEX); + pr_cont("\n"); +} + +static void local_lock_2(void) +{ + local_lock_acquire(&local_A); /* IRQ-ON */ + local_lock_release(&local_A); + + HARDIRQ_ENTER(); + spin_lock(&lock_A); /* IN-IRQ */ + spin_unlock(&lock_A); + HARDIRQ_EXIT() + + HARDIRQ_DISABLE(); + spin_lock(&lock_A); + local_lock_acquire(&local_A); /* IN-IRQ <-> IRQ-ON cycle, false */ + local_lock_release(&local_A); + spin_unlock(&lock_A); + HARDIRQ_ENABLE(); +} + +static void local_lock_3A(void) +{ + local_lock_acquire(&local_A); /* IRQ-ON */ + spin_lock(&lock_B); /* IRQ-ON */ + spin_unlock(&lock_B); + local_lock_release(&local_A); + + HARDIRQ_ENTER(); + spin_lock(&lock_A); /* IN-IRQ */ + spin_unlock(&lock_A); + HARDIRQ_EXIT() + + HARDIRQ_DISABLE(); + spin_lock(&lock_A); + local_lock_acquire(&local_A); /* IN-IRQ <-> IRQ-ON cycle only if we count local_lock(), false */ + local_lock_release(&local_A); + spin_unlock(&lock_A); + HARDIRQ_ENABLE(); +} + +static void local_lock_3B(void) +{ + local_lock_acquire(&local_A); /* IRQ-ON */ + spin_lock(&lock_B); /* IRQ-ON */ + spin_unlock(&lock_B); + local_lock_release(&local_A); + + HARDIRQ_ENTER(); + spin_lock(&lock_A); /* IN-IRQ */ + spin_unlock(&lock_A); + HARDIRQ_EXIT() + + HARDIRQ_DISABLE(); + spin_lock(&lock_A); + local_lock_acquire(&local_A); /* IN-IRQ <-> IRQ-ON cycle only if we count local_lock(), false */ + local_lock_release(&local_A); + spin_unlock(&lock_A); + HARDIRQ_ENABLE(); + + HARDIRQ_DISABLE(); + spin_lock(&lock_A); + spin_lock(&lock_B); /* IN-IRQ <-> IRQ-ON cycle, true */ + spin_unlock(&lock_B); + spin_unlock(&lock_A); + HARDIRQ_DISABLE(); + +} + +static void local_lock_tests(void) +{ + printk(" --------------------------------------------------------------------------\n"); + printk(" | local_lock tests |\n"); + printk(" ---------------------\n"); + + print_testname("local_lock inversion 2"); + dotest(local_lock_2, SUCCESS, LOCKTYPE_LL); + pr_cont("\n"); + + print_testname("local_lock inversion 3A"); + dotest(local_lock_3A, SUCCESS, LOCKTYPE_LL); + pr_cont("\n"); + + print_testname("local_lock inversion 3B"); + dotest(local_lock_3B, FAILURE, LOCKTYPE_LL); + pr_cont("\n"); +} + +static void hardirq_deadlock_softirq_not_deadlock(void) +{ + /* mutex_A is hardirq-unsafe and softirq-unsafe */ + /* mutex_A -> lock_C */ + mutex_lock(&mutex_A); + HARDIRQ_DISABLE(); + spin_lock(&lock_C); + spin_unlock(&lock_C); + HARDIRQ_ENABLE(); + mutex_unlock(&mutex_A); + + /* lock_A is hardirq-safe */ + HARDIRQ_ENTER(); + spin_lock(&lock_A); + spin_unlock(&lock_A); + HARDIRQ_EXIT(); + + /* lock_A -> lock_B */ + HARDIRQ_DISABLE(); + spin_lock(&lock_A); + spin_lock(&lock_B); + spin_unlock(&lock_B); + spin_unlock(&lock_A); + HARDIRQ_ENABLE(); + + /* lock_B -> lock_C */ + HARDIRQ_DISABLE(); + spin_lock(&lock_B); + spin_lock(&lock_C); + spin_unlock(&lock_C); + spin_unlock(&lock_B); + HARDIRQ_ENABLE(); + + /* lock_D is softirq-safe */ + SOFTIRQ_ENTER(); + spin_lock(&lock_D); + spin_unlock(&lock_D); + SOFTIRQ_EXIT(); + + /* And lock_D is hardirq-unsafe */ + SOFTIRQ_DISABLE(); + spin_lock(&lock_D); + spin_unlock(&lock_D); + SOFTIRQ_ENABLE(); + + /* + * mutex_A -> lock_C -> lock_D is softirq-unsafe -> softirq-safe, not + * deadlock. + * + * lock_A -> lock_B -> lock_C -> lock_D is hardirq-safe -> + * hardirq-unsafe, deadlock. + */ + HARDIRQ_DISABLE(); + spin_lock(&lock_C); + spin_lock(&lock_D); + spin_unlock(&lock_D); + spin_unlock(&lock_C); + HARDIRQ_ENABLE(); +} + void locking_selftest(void) { /* @@ -1979,6 +2802,11 @@ void locking_selftest(void) } /* + * treats read_lock() as recursive read locks for testing purpose + */ + force_read_lock_recursive = 1; + + /* * Run the testsuite: */ printk("------------------------\n"); @@ -1988,7 +2816,6 @@ void locking_selftest(void) printk(" --------------------------------------------------------------------------\n"); init_shared_classes(); - debug_locks_silent = !debug_locks_verbose; lockdep_set_selftest_task(current); DO_TESTCASE_6R("A-A deadlock", AA); @@ -2033,14 +2860,6 @@ void locking_selftest(void) print_testname("mixed read-lock/lock-write ABBA"); pr_cont(" |"); dotest(rlock_ABBA1, FAILURE, LOCKTYPE_RWLOCK); -#ifdef CONFIG_PROVE_LOCKING - /* - * Lockdep does indeed fail here, but there's nothing we can do about - * that now. Don't kill lockdep for it. - */ - unexpected_testcase_failures--; -#endif - pr_cont(" |"); dotest(rwsem_ABBA1, FAILURE, LOCKTYPE_RWSEM); @@ -2056,6 +2875,15 @@ void locking_selftest(void) pr_cont(" |"); dotest(rwsem_ABBA3, FAILURE, LOCKTYPE_RWSEM); + print_testname("chain cached mixed R-L/L-W ABBA"); + pr_cont(" |"); + dotest(rlock_chaincache_ABBA1, FAILURE, LOCKTYPE_RWLOCK); + + DO_TESTCASE_6x1RRB("rlock W1R2/W2R3/W3R1", W1R2_W2R3_W3R1); + DO_TESTCASE_6x1RRB("rlock W1W2/R2R3/W3R1", W1W2_R2R3_W3R1); + DO_TESTCASE_6x1RR("rlock W1W2/R2R3/R3W1", W1W2_R2R3_R3W1); + DO_TESTCASE_6x1RR("rlock W1R2/R2R3/W3W1", W1R2_R2R3_W3W1); + printk(" --------------------------------------------------------------------------\n"); /* @@ -2068,11 +2896,31 @@ void locking_selftest(void) DO_TESTCASE_6x6("safe-A + unsafe-B #2", irqsafe4); DO_TESTCASE_6x6RW("irq lock-inversion", irq_inversion); - DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion); -// DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2); + DO_TESTCASE_6x2x2RW("irq read-recursion", irq_read_recursion); + DO_TESTCASE_6x2x2RW("irq read-recursion #2", irq_read_recursion2); + DO_TESTCASE_6x2x2RW("irq read-recursion #3", irq_read_recursion3); ww_tests(); + force_read_lock_recursive = 0; + /* + * queued_read_lock() specific test cases can be put here + */ + if (IS_ENABLED(CONFIG_QUEUED_RWLOCKS)) + queued_read_lock_tests(); + + fs_reclaim_tests(); + + /* Wait context test cases that are specific for RAW_LOCK_NESTING */ + if (IS_ENABLED(CONFIG_PROVE_RAW_LOCK_NESTING)) + wait_context_tests(); + + local_lock_tests(); + + print_testname("hardirq_unsafe_softirq_safe"); + dotest(hardirq_deadlock_softirq_not_deadlock, FAILURE, LOCKTYPE_SPECIAL); + pr_cont("\n"); + if (unexpected_testcase_failures) { printk("-----------------------------------------------------------------\n"); debug_locks = 0; diff --git a/lib/logic_iomem.c b/lib/logic_iomem.c new file mode 100644 index 000000000000..9bdfde0c0f86 --- /dev/null +++ b/lib/logic_iomem.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Intel Corporation + * Author: Johannes Berg <johannes@sipsolutions.net> + */ +#include <linux/types.h> +#include <linux/slab.h> +#include <linux/logic_iomem.h> +#include <asm/io.h> + +struct logic_iomem_region { + const struct resource *res; + const struct logic_iomem_region_ops *ops; + struct list_head list; +}; + +struct logic_iomem_area { + const struct logic_iomem_ops *ops; + void *priv; +}; + +#define AREA_SHIFT 24 +#define MAX_AREA_SIZE (1 << AREA_SHIFT) +#define MAX_AREAS ((1ULL<<32) / MAX_AREA_SIZE) +#define AREA_BITS ((MAX_AREAS - 1) << AREA_SHIFT) +#define AREA_MASK (MAX_AREA_SIZE - 1) +#ifdef CONFIG_64BIT +#define IOREMAP_BIAS 0xDEAD000000000000UL +#define IOREMAP_MASK 0xFFFFFFFF00000000UL +#else +#define IOREMAP_BIAS 0 +#define IOREMAP_MASK 0 +#endif + +static DEFINE_MUTEX(regions_mtx); +static LIST_HEAD(regions_list); +static struct logic_iomem_area mapped_areas[MAX_AREAS]; + +int logic_iomem_add_region(struct resource *resource, + const struct logic_iomem_region_ops *ops) +{ + struct logic_iomem_region *rreg; + int err; + + if (WARN_ON(!resource || !ops)) + return -EINVAL; + + if (WARN_ON((resource->flags & IORESOURCE_TYPE_BITS) != IORESOURCE_MEM)) + return -EINVAL; + + rreg = kzalloc(sizeof(*rreg), GFP_KERNEL); + if (!rreg) + return -ENOMEM; + + err = request_resource(&iomem_resource, resource); + if (err) { + kfree(rreg); + return -ENOMEM; + } + + mutex_lock(®ions_mtx); + rreg->res = resource; + rreg->ops = ops; + list_add_tail(&rreg->list, ®ions_list); + mutex_unlock(®ions_mtx); + + return 0; +} +EXPORT_SYMBOL(logic_iomem_add_region); + +#ifndef CONFIG_LOGIC_IOMEM_FALLBACK +static void __iomem *real_ioremap(phys_addr_t offset, size_t size) +{ + WARN(1, "invalid ioremap(0x%llx, 0x%zx)\n", + (unsigned long long)offset, size); + return NULL; +} + +static void real_iounmap(void __iomem *addr) +{ + WARN(1, "invalid iounmap for addr 0x%llx\n", + (unsigned long long __force)addr); +} +#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */ + +void __iomem *ioremap(phys_addr_t offset, size_t size) +{ + void __iomem *ret = NULL; + struct logic_iomem_region *rreg, *found = NULL; + int i; + + mutex_lock(®ions_mtx); + list_for_each_entry(rreg, ®ions_list, list) { + if (rreg->res->start > offset) + continue; + if (rreg->res->end < offset + size - 1) + continue; + found = rreg; + break; + } + + if (!found) + goto out; + + for (i = 0; i < MAX_AREAS; i++) { + long offs; + + if (mapped_areas[i].ops) + continue; + + offs = rreg->ops->map(offset - found->res->start, + size, &mapped_areas[i].ops, + &mapped_areas[i].priv); + if (offs < 0) { + mapped_areas[i].ops = NULL; + break; + } + + if (WARN_ON(!mapped_areas[i].ops)) { + mapped_areas[i].ops = NULL; + break; + } + + ret = (void __iomem *)(IOREMAP_BIAS + (i << AREA_SHIFT) + offs); + break; + } +out: + mutex_unlock(®ions_mtx); + if (ret) + return ret; + return real_ioremap(offset, size); +} +EXPORT_SYMBOL(ioremap); + +static inline struct logic_iomem_area * +get_area(const volatile void __iomem *addr) +{ + unsigned long a = (unsigned long)addr; + unsigned int idx; + + if (WARN_ON((a & IOREMAP_MASK) != IOREMAP_BIAS)) + return NULL; + + idx = (a & AREA_BITS) >> AREA_SHIFT; + + if (mapped_areas[idx].ops) + return &mapped_areas[idx]; + + return NULL; +} + +void iounmap(void __iomem *addr) +{ + struct logic_iomem_area *area = get_area(addr); + + if (!area) { + real_iounmap(addr); + return; + } + + if (area->ops->unmap) + area->ops->unmap(area->priv); + + mutex_lock(®ions_mtx); + area->ops = NULL; + area->priv = NULL; + mutex_unlock(®ions_mtx); +} +EXPORT_SYMBOL(iounmap); + +#ifndef CONFIG_LOGIC_IOMEM_FALLBACK +#define MAKE_FALLBACK(op, sz) \ +static u##sz real_raw_read ## op(const volatile void __iomem *addr) \ +{ \ + WARN(1, "Invalid read" #op " at address %llx\n", \ + (unsigned long long __force)addr); \ + return (u ## sz)~0ULL; \ +} \ + \ +static void real_raw_write ## op(u ## sz val, \ + volatile void __iomem *addr) \ +{ \ + WARN(1, "Invalid writeq" #op " of 0x%llx at address %llx\n", \ + (unsigned long long)val, (unsigned long long __force)addr);\ +} \ + +MAKE_FALLBACK(b, 8); +MAKE_FALLBACK(w, 16); +MAKE_FALLBACK(l, 32); +#ifdef CONFIG_64BIT +MAKE_FALLBACK(q, 64); +#endif + +static void real_memset_io(volatile void __iomem *addr, int value, size_t size) +{ + WARN(1, "Invalid memset_io at address 0x%llx\n", + (unsigned long long __force)addr); +} + +static void real_memcpy_fromio(void *buffer, const volatile void __iomem *addr, + size_t size) +{ + WARN(1, "Invalid memcpy_fromio at address 0x%llx\n", + (unsigned long long __force)addr); + + memset(buffer, 0xff, size); +} + +static void real_memcpy_toio(volatile void __iomem *addr, const void *buffer, + size_t size) +{ + WARN(1, "Invalid memcpy_toio at address 0x%llx\n", + (unsigned long long __force)addr); +} +#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */ + +#define MAKE_OP(op, sz) \ +u##sz __raw_read ## op(const volatile void __iomem *addr) \ +{ \ + struct logic_iomem_area *area = get_area(addr); \ + \ + if (!area) \ + return real_raw_read ## op(addr); \ + \ + return (u ## sz) area->ops->read(area->priv, \ + (unsigned long)addr & AREA_MASK,\ + sz / 8); \ +} \ +EXPORT_SYMBOL(__raw_read ## op); \ + \ +void __raw_write ## op(u ## sz val, volatile void __iomem *addr) \ +{ \ + struct logic_iomem_area *area = get_area(addr); \ + \ + if (!area) { \ + real_raw_write ## op(val, addr); \ + return; \ + } \ + \ + area->ops->write(area->priv, \ + (unsigned long)addr & AREA_MASK, \ + sz / 8, val); \ +} \ +EXPORT_SYMBOL(__raw_write ## op) + +MAKE_OP(b, 8); +MAKE_OP(w, 16); +MAKE_OP(l, 32); +#ifdef CONFIG_64BIT +MAKE_OP(q, 64); +#endif + +void memset_io(volatile void __iomem *addr, int value, size_t size) +{ + struct logic_iomem_area *area = get_area(addr); + unsigned long offs, start; + + if (!area) { + real_memset_io(addr, value, size); + return; + } + + start = (unsigned long)addr & AREA_MASK; + + if (area->ops->set) { + area->ops->set(area->priv, start, value, size); + return; + } + + for (offs = 0; offs < size; offs++) + area->ops->write(area->priv, start + offs, 1, value); +} +EXPORT_SYMBOL(memset_io); + +void memcpy_fromio(void *buffer, const volatile void __iomem *addr, + size_t size) +{ + struct logic_iomem_area *area = get_area(addr); + u8 *buf = buffer; + unsigned long offs, start; + + if (!area) { + real_memcpy_fromio(buffer, addr, size); + return; + } + + start = (unsigned long)addr & AREA_MASK; + + if (area->ops->copy_from) { + area->ops->copy_from(area->priv, buffer, start, size); + return; + } + + for (offs = 0; offs < size; offs++) + buf[offs] = area->ops->read(area->priv, start + offs, 1); +} +EXPORT_SYMBOL(memcpy_fromio); + +void memcpy_toio(volatile void __iomem *addr, const void *buffer, size_t size) +{ + struct logic_iomem_area *area = get_area(addr); + const u8 *buf = buffer; + unsigned long offs, start; + + if (!area) { + real_memcpy_toio(addr, buffer, size); + return; + } + + start = (unsigned long)addr & AREA_MASK; + + if (area->ops->copy_to) { + area->ops->copy_to(area->priv, start, buffer, size); + return; + } + + for (offs = 0; offs < size; offs++) + area->ops->write(area->priv, start + offs, 1, buf[offs]); +} +EXPORT_SYMBOL(memcpy_toio); diff --git a/lib/logic_pio.c b/lib/logic_pio.c index f511a99bb389..07b4b9a1f54b 100644 --- a/lib/logic_pio.c +++ b/lib/logic_pio.c @@ -28,6 +28,8 @@ static DEFINE_MUTEX(io_range_mutex); * @new_range: pointer to the IO range to be registered. * * Returns 0 on success, the error code in case of failure. + * If the range already exists, -EEXIST will be returned, which should be + * considered a success. * * Register a new IO range node in the IO range list. */ @@ -51,6 +53,7 @@ int logic_pio_register_range(struct logic_pio_hwaddr *new_range) list_for_each_entry(range, &io_range_list, list) { if (range->fwnode == new_range->fwnode) { /* range already there */ + ret = -EEXIST; goto end_register; } if (range->flags == LOGIC_PIO_CPU_MMIO && @@ -229,13 +232,13 @@ unsigned long logic_pio_trans_cpuaddr(resource_size_t addr) } #if defined(CONFIG_INDIRECT_PIO) && defined(PCI_IOBASE) -#define BUILD_LOGIC_IO(bw, type) \ -type logic_in##bw(unsigned long addr) \ +#define BUILD_LOGIC_IO(bwl, type) \ +type logic_in##bwl(unsigned long addr) \ { \ type ret = (type)~0; \ \ if (addr < MMIO_UPPER_LIMIT) { \ - ret = read##bw(PCI_IOBASE + addr); \ + ret = _in##bwl(addr); \ } else if (addr >= MMIO_UPPER_LIMIT && addr < IO_SPACE_LIMIT) { \ struct logic_pio_hwaddr *entry = find_io_range(addr); \ \ @@ -248,10 +251,10 @@ type logic_in##bw(unsigned long addr) \ return ret; \ } \ \ -void logic_out##bw(type value, unsigned long addr) \ +void logic_out##bwl(type value, unsigned long addr) \ { \ if (addr < MMIO_UPPER_LIMIT) { \ - write##bw(value, PCI_IOBASE + addr); \ + _out##bwl(value, addr); \ } else if (addr >= MMIO_UPPER_LIMIT && addr < IO_SPACE_LIMIT) { \ struct logic_pio_hwaddr *entry = find_io_range(addr); \ \ @@ -263,11 +266,11 @@ void logic_out##bw(type value, unsigned long addr) \ } \ } \ \ -void logic_ins##bw(unsigned long addr, void *buffer, \ - unsigned int count) \ +void logic_ins##bwl(unsigned long addr, void *buffer, \ + unsigned int count) \ { \ if (addr < MMIO_UPPER_LIMIT) { \ - reads##bw(PCI_IOBASE + addr, buffer, count); \ + reads##bwl(PCI_IOBASE + addr, buffer, count); \ } else if (addr >= MMIO_UPPER_LIMIT && addr < IO_SPACE_LIMIT) { \ struct logic_pio_hwaddr *entry = find_io_range(addr); \ \ @@ -280,11 +283,11 @@ void logic_ins##bw(unsigned long addr, void *buffer, \ \ } \ \ -void logic_outs##bw(unsigned long addr, const void *buffer, \ - unsigned int count) \ +void logic_outs##bwl(unsigned long addr, const void *buffer, \ + unsigned int count) \ { \ if (addr < MMIO_UPPER_LIMIT) { \ - writes##bw(PCI_IOBASE + addr, buffer, count); \ + writes##bwl(PCI_IOBASE + addr, buffer, count); \ } else if (addr >= MMIO_UPPER_LIMIT && addr < IO_SPACE_LIMIT) { \ struct logic_pio_hwaddr *entry = find_io_range(addr); \ \ diff --git a/lib/lru_cache.c b/lib/lru_cache.c index c69ee53d8dde..52313acbfa62 100644 --- a/lib/lru_cache.c +++ b/lib/lru_cache.c @@ -76,6 +76,7 @@ int lc_try_lock(struct lru_cache *lc) /** * lc_create - prepares to track objects in an active set * @name: descriptive name only used in lc_seq_printf_stats and lc_seq_dump_details + * @cache: cache root pointer * @max_pending_changes: maximum changes to accumulate until a transaction is required * @e_count: number of elements allowed to be active simultaneously * @e_size: size of the tracked objects @@ -627,7 +628,7 @@ void lc_set(struct lru_cache *lc, unsigned int enr, int index) } /** - * lc_dump - Dump a complete LRU cache to seq in textual form. + * lc_seq_dump_details - Dump a complete LRU cache to seq in textual form. * @lc: the lru cache to operate on * @seq: the &struct seq_file pointer to seq_printf into * @utext: user supplied additional "heading" or other info diff --git a/lib/lz4/lz4_compress.c b/lib/lz4/lz4_compress.c index cc7b6d4cc7c7..90bb67994688 100644 --- a/lib/lz4/lz4_compress.c +++ b/lib/lz4/lz4_compress.c @@ -446,7 +446,7 @@ _last_literals: *op++ = (BYTE)(lastRun << ML_BITS); } - memcpy(op, anchor, lastRun); + LZ4_memcpy(op, anchor, lastRun); op += lastRun; } @@ -708,7 +708,7 @@ _last_literals: } else { *op++ = (BYTE)(lastRunSize<<ML_BITS); } - memcpy(op, anchor, lastRunSize); + LZ4_memcpy(op, anchor, lastRunSize); op += lastRunSize; } diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c index 0c9d3ad17e0f..926f4823d5ea 100644 --- a/lib/lz4/lz4_decompress.c +++ b/lib/lz4/lz4_decompress.c @@ -141,6 +141,9 @@ static FORCE_INLINE int LZ4_decompress_generic( * space in the output for those 18 bytes earlier, upon * entering the shortcut (in other words, there is a * combined check for both stages). + * + * The & in the likely() below is intentionally not && so that + * some compilers can produce better parallelized runtime code */ if ((endOnInput ? length != RUN_MASK : length <= 8) /* @@ -150,7 +153,7 @@ static FORCE_INLINE int LZ4_decompress_generic( && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend))) { /* Copy the literals */ - memcpy(op, ip, endOnInput ? 16 : 8); + LZ4_memcpy(op, ip, endOnInput ? 16 : 8); op += length; ip += length; /* @@ -169,9 +172,9 @@ static FORCE_INLINE int LZ4_decompress_generic( (offset >= 8) && (dict == withPrefix64k || match >= lowPrefix)) { /* Copy the match. */ - memcpy(op + 0, match + 0, 8); - memcpy(op + 8, match + 8, 8); - memcpy(op + 16, match + 16, 2); + LZ4_memcpy(op + 0, match + 0, 8); + LZ4_memcpy(op + 8, match + 8, 8); + LZ4_memcpy(op + 16, match + 16, 2); op += length + MINMATCH; /* Both stages worked, load the next token. */ continue; @@ -260,7 +263,11 @@ static FORCE_INLINE int LZ4_decompress_generic( } } - memcpy(op, ip, length); + /* + * supports overlapping memory regions; only matters + * for in-place decompression scenarios + */ + LZ4_memmove(op, ip, length); ip += length; op += length; @@ -347,7 +354,7 @@ _copy_match: size_t const copySize = (size_t)(lowPrefix - match); size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); + LZ4_memcpy(op, dictEnd - copySize, copySize); op += copySize; if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ @@ -357,7 +364,7 @@ _copy_match: while (op < endOfMatch) *op++ = *copyFrom++; } else { - memcpy(op, lowPrefix, restSize); + LZ4_memcpy(op, lowPrefix, restSize); op += restSize; } } @@ -383,7 +390,7 @@ _copy_match: while (op < copyEnd) *op++ = *match++; } else { - memcpy(op, match, mlen); + LZ4_memcpy(op, match, mlen); } op = copyEnd; if (op == oend) @@ -397,7 +404,7 @@ _copy_match: op[2] = match[2]; op[3] = match[3]; match += inc32table[offset]; - memcpy(op + 4, match, 4); + LZ4_memcpy(op + 4, match, 4); match -= dec64table[offset]; } else { LZ4_copy8(op, match); @@ -474,7 +481,7 @@ int LZ4_decompress_fast(const char *source, char *dest, int originalSize) /* ===== Instantiate a few more decoding cases, used more than once. ===== */ -int LZ4_decompress_safe_withPrefix64k(const char *source, char *dest, +static int LZ4_decompress_safe_withPrefix64k(const char *source, char *dest, int compressedSize, int maxOutputSize) { return LZ4_decompress_generic(source, dest, diff --git a/lib/lz4/lz4defs.h b/lib/lz4/lz4defs.h index 1a7fa9d9170f..673bd206aa98 100644 --- a/lib/lz4/lz4defs.h +++ b/lib/lz4/lz4defs.h @@ -137,6 +137,17 @@ static FORCE_INLINE void LZ4_writeLE16(void *memPtr, U16 value) return put_unaligned_le16(value, memPtr); } +/* + * LZ4 relies on memcpy with a constant size being inlined. In freestanding + * environments, the compiler can't assume the implementation of memcpy() is + * standard compliant, so apply its specialized memcpy() inlining logic. When + * possible, use __builtin_memcpy() to tell the compiler to analyze memcpy() + * as-if it were standard compliant, so it can inline it in freestanding + * environments. This is needed when decompressing the Linux Kernel, for example. + */ +#define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +#define LZ4_memmove(dst, src, size) __builtin_memmove(dst, src, size) + static FORCE_INLINE void LZ4_copy8(void *dst, const void *src) { #if LZ4_ARCH64 diff --git a/lib/lz4/lz4hc_compress.c b/lib/lz4/lz4hc_compress.c index 1b61d874e337..e7ac8694b797 100644 --- a/lib/lz4/lz4hc_compress.c +++ b/lib/lz4/lz4hc_compress.c @@ -570,7 +570,7 @@ _Search3: *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun<<ML_BITS); - memcpy(op, anchor, iend - anchor); + LZ4_memcpy(op, anchor, iend - anchor); op += iend - anchor; } diff --git a/lib/lzo/lzo1x_compress.c b/lib/lzo/lzo1x_compress.c index 717c940112f9..76758e9296ba 100644 --- a/lib/lzo/lzo1x_compress.c +++ b/lib/lzo/lzo1x_compress.c @@ -268,6 +268,19 @@ m_len_done: *op++ = (M4_MARKER | ((m_off >> 11) & 8) | (m_len - 2)); else { + if (unlikely(((m_off & 0x403f) == 0x403f) + && (m_len >= 261) + && (m_len <= 264)) + && likely(bitstream_version)) { + // Under lzo-rle, block copies + // for 261 <= length <= 264 and + // (distance & 0x80f3) == 0x80f3 + // can result in ambiguous + // output. Adjust length + // to 260 to prevent ambiguity. + ip -= m_len - 260; + m_len = 260; + } m_len -= M4_MAX_LEN; *op++ = (M4_MARKER | ((m_off >> 11) & 8)); while (unlikely(m_len > 255)) { @@ -288,7 +301,7 @@ finished_writing_instruction: return in_end - (ii - ti); } -int lzogeneric1x_1_compress(const unsigned char *in, size_t in_len, +static int lzogeneric1x_1_compress(const unsigned char *in, size_t in_len, unsigned char *out, size_t *out_len, void *wrkmem, const unsigned char bitstream_version) { diff --git a/lib/lzo/lzo1x_decompress_safe.c b/lib/lzo/lzo1x_decompress_safe.c index 2717c7963acd..7892a40cf765 100644 --- a/lib/lzo/lzo1x_decompress_safe.c +++ b/lib/lzo/lzo1x_decompress_safe.c @@ -32,7 +32,7 @@ * depending on the base count. Since the base count is taken from a u8 * and a few bits, it is safe to assume that it will always be lower than * or equal to 2*255, thus we can always prevent any overflow by accepting - * two less 255 steps. See Documentation/lzo.txt for more information. + * two less 255 steps. See Documentation/staging/lzo.rst for more information. */ #define MAX_255_COUNT ((((size_t)~0) / 255) - 2) diff --git a/lib/math/Kconfig b/lib/math/Kconfig index 15bd50d92308..0634b428d0cb 100644 --- a/lib/math/Kconfig +++ b/lib/math/Kconfig @@ -6,7 +6,12 @@ config CORDIC calculations are in fixed point. Module will be called cordic. config PRIME_NUMBERS - tristate + tristate "Simple prime number generator for testing" + help + This option provides a simple prime number generator for test + modules. + + If unsure, say N. config RATIONAL - bool + tristate diff --git a/lib/math/Makefile b/lib/math/Makefile index be6909e943bd..bfac26ddfc22 100644 --- a/lib/math/Makefile +++ b/lib/math/Makefile @@ -4,3 +4,6 @@ obj-y += div64.o gcd.o lcm.o int_pow.o int_sqrt.o reciprocal_div.o obj-$(CONFIG_CORDIC) += cordic.o obj-$(CONFIG_PRIME_NUMBERS) += prime_numbers.o obj-$(CONFIG_RATIONAL) += rational.o + +obj-$(CONFIG_TEST_DIV64) += test_div64.o +obj-$(CONFIG_RATIONAL_KUNIT_TEST) += rational-test.o diff --git a/lib/math/div64.c b/lib/math/div64.c index 368ca7fd0d82..46866394fc84 100644 --- a/lib/math/div64.c +++ b/lib/math/div64.c @@ -18,9 +18,11 @@ * or by defining a preprocessor macro in arch/include/asm/div64.h. */ +#include <linux/bitops.h> #include <linux/export.h> -#include <linux/kernel.h> +#include <linux/math.h> #include <linux/math64.h> +#include <linux/log2.h> /* Not needed on 64bit architectures */ #if BITS_PER_LONG == 32 @@ -190,3 +192,45 @@ u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder) return __iter_div_u64_rem(dividend, divisor, remainder); } EXPORT_SYMBOL(iter_div_u64_rem); + +#ifndef mul_u64_u64_div_u64 +u64 mul_u64_u64_div_u64(u64 a, u64 b, u64 c) +{ + u64 res = 0, div, rem; + int shift; + + /* can a * b overflow ? */ + if (ilog2(a) + ilog2(b) > 62) { + /* + * (b * a) / c is equal to + * + * (b / c) * a + + * (b % c) * a / c + * + * if nothing overflows. Can the 1st multiplication + * overflow? Yes, but we do not care: this can only + * happen if the end result can't fit in u64 anyway. + * + * So the code below does + * + * res = (b / c) * a; + * b = b % c; + */ + div = div64_u64_rem(b, c, &rem); + res = div * a; + b = rem; + + shift = ilog2(a) + ilog2(b) - 62; + if (shift > 0) { + /* drop precision */ + b >>= shift; + c >>= shift; + if (!c) + return res; + } + } + + return res + div64_u64(a * b, c); +} +EXPORT_SYMBOL(mul_u64_u64_div_u64); +#endif diff --git a/lib/math/int_pow.c b/lib/math/int_pow.c index 622fc1ab3c74..0cf426e69bda 100644 --- a/lib/math/int_pow.c +++ b/lib/math/int_pow.c @@ -6,7 +6,7 @@ */ #include <linux/export.h> -#include <linux/kernel.h> +#include <linux/math.h> #include <linux/types.h> /** diff --git a/lib/math/int_sqrt.c b/lib/math/int_sqrt.c index 30e0f9770f88..a8170bb9142f 100644 --- a/lib/math/int_sqrt.c +++ b/lib/math/int_sqrt.c @@ -6,9 +6,10 @@ * square root from Guy L. Steele. */ -#include <linux/kernel.h> #include <linux/export.h> #include <linux/bitops.h> +#include <linux/limits.h> +#include <linux/math.h> /** * int_sqrt - computes the integer square root diff --git a/lib/math/prime_numbers.c b/lib/math/prime_numbers.c index 052f5b727be7..d42cebf7407f 100644 --- a/lib/math/prime_numbers.c +++ b/lib/math/prime_numbers.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -#define pr_fmt(fmt) "prime numbers: " fmt "\n" +#define pr_fmt(fmt) "prime numbers: " fmt #include <linux/module.h> #include <linux/mutex.h> @@ -253,7 +253,7 @@ static void dump_primes(void) if (buf) bitmap_print_to_pagebuf(true, buf, p->primes, p->sz); - pr_info("primes.{last=%lu, .sz=%lu, .primes[]=...x%lx} = %s", + pr_info("primes.{last=%lu, .sz=%lu, .primes[]=...x%lx} = %s\n", p->last, p->sz, p->primes[BITS_TO_LONGS(p->sz) - 1], buf); rcu_read_unlock(); @@ -273,7 +273,7 @@ static int selftest(unsigned long max) bool fast = is_prime_number(x); if (slow != fast) { - pr_err("inconsistent result for is-prime(%lu): slow=%s, fast=%s!", + pr_err("inconsistent result for is-prime(%lu): slow=%s, fast=%s!\n", x, slow ? "yes" : "no", fast ? "yes" : "no"); goto err; } @@ -282,14 +282,14 @@ static int selftest(unsigned long max) continue; if (next_prime_number(last) != x) { - pr_err("incorrect result for next-prime(%lu): expected %lu, got %lu", + pr_err("incorrect result for next-prime(%lu): expected %lu, got %lu\n", last, x, next_prime_number(last)); goto err; } last = x; } - pr_info("selftest(%lu) passed, last prime was %lu", x, last); + pr_info("%s(%lu) passed, last prime was %lu\n", __func__, x, last); return 0; err: diff --git a/lib/math/rational-test.c b/lib/math/rational-test.c new file mode 100644 index 000000000000..01611ddff420 --- /dev/null +++ b/lib/math/rational-test.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <kunit/test.h> + +#include <linux/rational.h> + +struct rational_test_param { + unsigned long num, den; + unsigned long max_num, max_den; + unsigned long exp_num, exp_den; + + const char *name; +}; + +static const struct rational_test_param test_parameters[] = { + { 1230, 10, 100, 20, 100, 1, "Exceeds bounds, semi-convergent term > 1/2 last term" }, + { 34567,100, 120, 20, 120, 1, "Exceeds bounds, semi-convergent term < 1/2 last term" }, + { 1, 30, 100, 10, 0, 1, "Closest to zero" }, + { 1, 19, 100, 10, 1, 10, "Closest to smallest non-zero" }, + { 27,32, 16, 16, 11, 13, "Use convergent" }, + { 1155, 7735, 255, 255, 33, 221, "Exact answer" }, + { 87, 32, 70, 32, 68, 25, "Semiconvergent, numerator limit" }, + { 14533, 4626, 15000, 2400, 7433, 2366, "Semiconvergent, denominator limit" }, +}; + +static void get_desc(const struct rational_test_param *param, char *desc) +{ + strscpy(desc, param->name, KUNIT_PARAM_DESC_SIZE); +} + +/* Creates function rational_gen_params */ +KUNIT_ARRAY_PARAM(rational, test_parameters, get_desc); + +static void rational_test(struct kunit *test) +{ + const struct rational_test_param *param = (const struct rational_test_param *)test->param_value; + unsigned long n = 0, d = 0; + + rational_best_approximation(param->num, param->den, param->max_num, param->max_den, &n, &d); + KUNIT_EXPECT_EQ(test, n, param->exp_num); + KUNIT_EXPECT_EQ(test, d, param->exp_den); +} + +static struct kunit_case rational_test_cases[] = { + KUNIT_CASE_PARAM(rational_test, rational_gen_params), + {} +}; + +static struct kunit_suite rational_test_suite = { + .name = "rational", + .test_cases = rational_test_cases, +}; + +kunit_test_suites(&rational_test_suite); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/math/rational.c b/lib/math/rational.c index 31fb27db2deb..ec59d426ea63 100644 --- a/lib/math/rational.c +++ b/lib/math/rational.c @@ -11,7 +11,9 @@ #include <linux/rational.h> #include <linux/compiler.h> #include <linux/export.h> -#include <linux/kernel.h> +#include <linux/minmax.h> +#include <linux/limits.h> +#include <linux/module.h> /* * calculate best rational approximation for a given fraction @@ -27,7 +29,7 @@ * with the fractional part size described in given_denominator. * * for theoretical background, see: - * http://en.wikipedia.org/wiki/Continued_fraction + * https://en.wikipedia.org/wiki/Continued_fraction */ void rational_best_approximation( @@ -78,13 +80,18 @@ void rational_best_approximation( * found below as 't'. */ if ((n2 > max_numerator) || (d2 > max_denominator)) { - unsigned long t = min((max_numerator - n0) / n1, - (max_denominator - d0) / d1); + unsigned long t = ULONG_MAX; - /* This tests if the semi-convergent is closer - * than the previous convergent. + if (d1) + t = (max_denominator - d0) / d1; + if (n1) + t = min(t, (max_numerator - n0) / n1); + + /* This tests if the semi-convergent is closer than the previous + * convergent. If d1 is zero there is no previous convergent as this + * is the 1st iteration, so always choose the semi-convergent. */ - if (2u * t > a || (2u * t == a && d0 * dp > d1 * d)) { + if (!d1 || 2u * t > a || (2u * t == a && d0 * dp > d1 * d)) { n1 = n0 + t * n1; d1 = d0 + t * d1; } @@ -100,3 +107,5 @@ void rational_best_approximation( } EXPORT_SYMBOL(rational_best_approximation); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/math/reciprocal_div.c b/lib/math/reciprocal_div.c index bf043258fa00..6cb4adbb81d2 100644 --- a/lib/math/reciprocal_div.c +++ b/lib/math/reciprocal_div.c @@ -1,9 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/bitops.h> #include <linux/bug.h> -#include <linux/kernel.h> -#include <asm/div64.h> -#include <linux/reciprocal_div.h> #include <linux/export.h> +#include <linux/limits.h> +#include <linux/math.h> +#include <linux/minmax.h> +#include <linux/types.h> + +#include <linux/reciprocal_div.h> /* * For a description of the algorithm please have a look at diff --git a/lib/math/test_div64.c b/lib/math/test_div64.c new file mode 100644 index 000000000000..c15edd688dd2 --- /dev/null +++ b/lib/math/test_div64.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Maciej W. Rozycki + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/init.h> +#include <linux/ktime.h> +#include <linux/module.h> +#include <linux/printk.h> +#include <linux/time64.h> +#include <linux/types.h> + +#include <asm/div64.h> + +#define TEST_DIV64_N_ITER 1024 + +static const u64 test_div64_dividends[] = { + 0x00000000ab275080, + 0x0000000fe73c1959, + 0x000000e54c0a74b1, + 0x00000d4398ff1ef9, + 0x0000a18c2ee1c097, + 0x00079fb80b072e4a, + 0x0072db27380dd689, + 0x0842f488162e2284, + 0xf66745411d8ab063, +}; +#define SIZE_DIV64_DIVIDENDS ARRAY_SIZE(test_div64_dividends) + +#define TEST_DIV64_DIVISOR_0 0x00000009 +#define TEST_DIV64_DIVISOR_1 0x0000007c +#define TEST_DIV64_DIVISOR_2 0x00000204 +#define TEST_DIV64_DIVISOR_3 0x0000cb5b +#define TEST_DIV64_DIVISOR_4 0x00010000 +#define TEST_DIV64_DIVISOR_5 0x0008a880 +#define TEST_DIV64_DIVISOR_6 0x003fd3ae +#define TEST_DIV64_DIVISOR_7 0x0b658fac +#define TEST_DIV64_DIVISOR_8 0xdc08b349 + +static const u32 test_div64_divisors[] = { + TEST_DIV64_DIVISOR_0, + TEST_DIV64_DIVISOR_1, + TEST_DIV64_DIVISOR_2, + TEST_DIV64_DIVISOR_3, + TEST_DIV64_DIVISOR_4, + TEST_DIV64_DIVISOR_5, + TEST_DIV64_DIVISOR_6, + TEST_DIV64_DIVISOR_7, + TEST_DIV64_DIVISOR_8, +}; +#define SIZE_DIV64_DIVISORS ARRAY_SIZE(test_div64_divisors) + +static const struct { + u64 quotient; + u32 remainder; +} test_div64_results[SIZE_DIV64_DIVISORS][SIZE_DIV64_DIVIDENDS] = { + { + { 0x0000000013045e47, 0x00000001 }, + { 0x000000000161596c, 0x00000030 }, + { 0x000000000054e9d4, 0x00000130 }, + { 0x000000000000d776, 0x0000278e }, + { 0x000000000000ab27, 0x00005080 }, + { 0x00000000000013c4, 0x0004ce80 }, + { 0x00000000000002ae, 0x001e143c }, + { 0x000000000000000f, 0x0033e56c }, + { 0x0000000000000000, 0xab275080 }, + }, { + { 0x00000001c45c02d1, 0x00000000 }, + { 0x0000000020d5213c, 0x00000049 }, + { 0x0000000007e3d65f, 0x000001dd }, + { 0x0000000000140531, 0x000065ee }, + { 0x00000000000fe73c, 0x00001959 }, + { 0x000000000001d637, 0x0004e5d9 }, + { 0x0000000000003fc9, 0x000713bb }, + { 0x0000000000000165, 0x029abe7d }, + { 0x0000000000000012, 0x6e9f7e37 }, + }, { + { 0x000000197a3a0cf7, 0x00000002 }, + { 0x00000001d9632e5c, 0x00000021 }, + { 0x0000000071c28039, 0x000001cd }, + { 0x000000000120a844, 0x0000b885 }, + { 0x0000000000e54c0a, 0x000074b1 }, + { 0x00000000001a7bb3, 0x00072331 }, + { 0x00000000000397ad, 0x0002c61b }, + { 0x000000000000141e, 0x06ea2e89 }, + { 0x000000000000010a, 0xab002ad7 }, + }, { + { 0x0000017949e37538, 0x00000001 }, + { 0x0000001b62441f37, 0x00000055 }, + { 0x0000000694a3391d, 0x00000085 }, + { 0x0000000010b2a5d2, 0x0000a753 }, + { 0x000000000d4398ff, 0x00001ef9 }, + { 0x0000000001882ec6, 0x0005cbf9 }, + { 0x000000000035333b, 0x0017abdf }, + { 0x00000000000129f1, 0x0ab4520d }, + { 0x0000000000000f6e, 0x8ac0ce9b }, + }, { + { 0x000011f321a74e49, 0x00000006 }, + { 0x0000014d8481d211, 0x0000005b }, + { 0x0000005025cbd92d, 0x000001e3 }, + { 0x00000000cb5e71e3, 0x000043e6 }, + { 0x00000000a18c2ee1, 0x0000c097 }, + { 0x0000000012a88828, 0x00036c97 }, + { 0x000000000287f16f, 0x002c2a25 }, + { 0x00000000000e2cc7, 0x02d581e3 }, + { 0x000000000000bbf4, 0x1ba08c03 }, + }, { + { 0x0000d8db8f72935d, 0x00000005 }, + { 0x00000fbd5aed7a2e, 0x00000002 }, + { 0x000003c84b6ea64a, 0x00000122 }, + { 0x0000000998fa8829, 0x000044b7 }, + { 0x000000079fb80b07, 0x00002e4a }, + { 0x00000000e16b20fa, 0x0002a14a }, + { 0x000000001e940d22, 0x00353b2e }, + { 0x0000000000ab40ac, 0x06fba6ba }, + { 0x000000000008debd, 0x72d98365 }, + }, { + { 0x000cc3045b8fc281, 0x00000000 }, + { 0x0000ed1f48b5c9fc, 0x00000079 }, + { 0x000038fb9c63406a, 0x000000e1 }, + { 0x000000909705b825, 0x00000a62 }, + { 0x00000072db27380d, 0x0000d689 }, + { 0x0000000d43fce827, 0x00082b09 }, + { 0x00000001ccaba11a, 0x0037e8dd }, + { 0x000000000a13f729, 0x0566dffd }, + { 0x000000000085a14b, 0x23d36726 }, + }, { + { 0x00eafeb9c993592b, 0x00000001 }, + { 0x00110e5befa9a991, 0x00000048 }, + { 0x00041947b4a1d36a, 0x000000dc }, + { 0x00000a6679327311, 0x0000c079 }, + { 0x00000842f488162e, 0x00002284 }, + { 0x000000f4459740fc, 0x00084484 }, + { 0x0000002122c47bf9, 0x002ca446 }, + { 0x00000000b9936290, 0x004979c4 }, + { 0x00000000099ca89d, 0x9db446bf }, + }, { + { 0x1b60cece589da1d2, 0x00000001 }, + { 0x01fcb42be1453f5b, 0x0000004f }, + { 0x007a3f2457df0749, 0x0000013f }, + { 0x0001363130e3ec7b, 0x000017aa }, + { 0x0000f66745411d8a, 0x0000b063 }, + { 0x00001c757dfab350, 0x00048863 }, + { 0x000003dc4979c652, 0x00224ea7 }, + { 0x000000159edc3144, 0x06409ab3 }, + { 0x000000011eadfee3, 0xa99c48a8 }, + }, +}; + +static inline bool test_div64_verify(u64 quotient, u32 remainder, int i, int j) +{ + return (quotient == test_div64_results[i][j].quotient && + remainder == test_div64_results[i][j].remainder); +} + +/* + * This needs to be a macro, because we don't want to rely on the compiler + * to do constant propagation, and `do_div' may take a different path for + * constants, so we do want to verify that as well. + */ +#define test_div64_one(dividend, divisor, i, j) ({ \ + bool result = true; \ + u64 quotient; \ + u32 remainder; \ + \ + quotient = dividend; \ + remainder = do_div(quotient, divisor); \ + if (!test_div64_verify(quotient, remainder, i, j)) { \ + pr_err("ERROR: %016llx / %08x => %016llx,%08x\n", \ + dividend, divisor, quotient, remainder); \ + pr_err("ERROR: expected value => %016llx,%08x\n",\ + test_div64_results[i][j].quotient, \ + test_div64_results[i][j].remainder); \ + result = false; \ + } \ + result; \ +}) + +/* + * Run calculation for the same divisor value expressed as a constant + * and as a variable, so as to verify the implementation for both cases + * should they be handled by different code execution paths. + */ +static bool __init test_div64(void) +{ + u64 dividend; + int i, j; + + for (i = 0; i < SIZE_DIV64_DIVIDENDS; i++) { + dividend = test_div64_dividends[i]; + if (!test_div64_one(dividend, TEST_DIV64_DIVISOR_0, i, 0)) + return false; + if (!test_div64_one(dividend, TEST_DIV64_DIVISOR_1, i, 1)) + return false; + if (!test_div64_one(dividend, TEST_DIV64_DIVISOR_2, i, 2)) + return false; + if (!test_div64_one(dividend, TEST_DIV64_DIVISOR_3, i, 3)) + return false; + if (!test_div64_one(dividend, TEST_DIV64_DIVISOR_4, i, 4)) + return false; + if (!test_div64_one(dividend, TEST_DIV64_DIVISOR_5, i, 5)) + return false; + if (!test_div64_one(dividend, TEST_DIV64_DIVISOR_6, i, 6)) + return false; + if (!test_div64_one(dividend, TEST_DIV64_DIVISOR_7, i, 7)) + return false; + if (!test_div64_one(dividend, TEST_DIV64_DIVISOR_8, i, 8)) + return false; + for (j = 0; j < SIZE_DIV64_DIVISORS; j++) { + if (!test_div64_one(dividend, test_div64_divisors[j], + i, j)) + return false; + } + } + return true; +} + +static int __init test_div64_init(void) +{ + struct timespec64 ts, ts0, ts1; + int i; + + pr_info("Starting 64bit/32bit division and modulo test\n"); + ktime_get_ts64(&ts0); + + for (i = 0; i < TEST_DIV64_N_ITER; i++) + if (!test_div64()) + break; + + ktime_get_ts64(&ts1); + ts = timespec64_sub(ts1, ts0); + pr_info("Completed 64bit/32bit division and modulo test, " + "%llu.%09lus elapsed\n", ts.tv_sec, ts.tv_nsec); + + return 0; +} + +static void __exit test_div64_exit(void) +{ +} + +module_init(test_div64_init); +module_exit(test_div64_exit); + +MODULE_AUTHOR("Maciej W. Rozycki <macro@orcam.me.uk>"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("64bit/32bit division and modulo test module"); diff --git a/lib/memcpy_kunit.c b/lib/memcpy_kunit.c new file mode 100644 index 000000000000..62f8ffcbbaa3 --- /dev/null +++ b/lib/memcpy_kunit.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test cases for memcpy(), memmove(), and memset(). + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <kunit/test.h> +#include <linux/device.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/overflow.h> +#include <linux/slab.h> +#include <linux/types.h> +#include <linux/vmalloc.h> + +struct some_bytes { + union { + u8 data[32]; + struct { + u32 one; + u16 two; + u8 three; + /* 1 byte hole */ + u32 four[4]; + }; + }; +}; + +#define check(instance, v) do { \ + int i; \ + BUILD_BUG_ON(sizeof(instance.data) != 32); \ + for (i = 0; i < sizeof(instance.data); i++) { \ + KUNIT_ASSERT_EQ_MSG(test, instance.data[i], v, \ + "line %d: '%s' not initialized to 0x%02x @ %d (saw 0x%02x)\n", \ + __LINE__, #instance, v, i, instance.data[i]); \ + } \ +} while (0) + +#define compare(name, one, two) do { \ + int i; \ + BUILD_BUG_ON(sizeof(one) != sizeof(two)); \ + for (i = 0; i < sizeof(one); i++) { \ + KUNIT_EXPECT_EQ_MSG(test, one.data[i], two.data[i], \ + "line %d: %s.data[%d] (0x%02x) != %s.data[%d] (0x%02x)\n", \ + __LINE__, #one, i, one.data[i], #two, i, two.data[i]); \ + } \ + kunit_info(test, "ok: " TEST_OP "() " name "\n"); \ +} while (0) + +static void memcpy_test(struct kunit *test) +{ +#define TEST_OP "memcpy" + struct some_bytes control = { + .data = { 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + }, + }; + struct some_bytes zero = { }; + struct some_bytes middle = { + .data = { 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + }, + }; + struct some_bytes three = { + .data = { 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + }, + }; + struct some_bytes dest = { }; + int count; + u8 *ptr; + + /* Verify static initializers. */ + check(control, 0x20); + check(zero, 0); + compare("static initializers", dest, zero); + + /* Verify assignment. */ + dest = control; + compare("direct assignment", dest, control); + + /* Verify complete overwrite. */ + memcpy(dest.data, zero.data, sizeof(dest.data)); + compare("complete overwrite", dest, zero); + + /* Verify middle overwrite. */ + dest = control; + memcpy(dest.data + 12, zero.data, 7); + compare("middle overwrite", dest, middle); + + /* Verify argument side-effects aren't repeated. */ + dest = control; + ptr = dest.data; + count = 1; + memcpy(ptr++, zero.data, count++); + ptr += 8; + memcpy(ptr++, zero.data, count++); + compare("argument side-effects", dest, three); +#undef TEST_OP +} + +static void memmove_test(struct kunit *test) +{ +#define TEST_OP "memmove" + struct some_bytes control = { + .data = { 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + }, + }; + struct some_bytes zero = { }; + struct some_bytes middle = { + .data = { 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + }, + }; + struct some_bytes five = { + .data = { 0x00, 0x00, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x00, 0x00, 0x00, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + }, + }; + struct some_bytes overlap = { + .data = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + }, + }; + struct some_bytes overlap_expected = { + .data = { 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x04, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + }, + }; + struct some_bytes dest = { }; + int count; + u8 *ptr; + + /* Verify static initializers. */ + check(control, 0x99); + check(zero, 0); + compare("static initializers", zero, dest); + + /* Verify assignment. */ + dest = control; + compare("direct assignment", dest, control); + + /* Verify complete overwrite. */ + memmove(dest.data, zero.data, sizeof(dest.data)); + compare("complete overwrite", dest, zero); + + /* Verify middle overwrite. */ + dest = control; + memmove(dest.data + 12, zero.data, 7); + compare("middle overwrite", dest, middle); + + /* Verify argument side-effects aren't repeated. */ + dest = control; + ptr = dest.data; + count = 2; + memmove(ptr++, zero.data, count++); + ptr += 9; + memmove(ptr++, zero.data, count++); + compare("argument side-effects", dest, five); + + /* Verify overlapping overwrite is correct. */ + ptr = &overlap.data[2]; + memmove(ptr, overlap.data, 5); + compare("overlapping write", overlap, overlap_expected); +#undef TEST_OP +} + +static void memset_test(struct kunit *test) +{ +#define TEST_OP "memset" + struct some_bytes control = { + .data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + }, + }; + struct some_bytes complete = { + .data = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }, + }; + struct some_bytes middle = { + .data = { 0x30, 0x30, 0x30, 0x30, 0x31, 0x31, 0x31, 0x31, + 0x31, 0x31, 0x31, 0x31, 0x31, 0x31, 0x31, 0x31, + 0x31, 0x31, 0x31, 0x31, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + }, + }; + struct some_bytes three = { + .data = { 0x60, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x61, 0x61, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + }, + }; + struct some_bytes after = { + .data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x72, + 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, + 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, + 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, + }, + }; + struct some_bytes startat = { + .data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, + 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, + 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, + }, + }; + struct some_bytes dest = { }; + int count, value; + u8 *ptr; + + /* Verify static initializers. */ + check(control, 0x30); + check(dest, 0); + + /* Verify assignment. */ + dest = control; + compare("direct assignment", dest, control); + + /* Verify complete overwrite. */ + memset(dest.data, 0xff, sizeof(dest.data)); + compare("complete overwrite", dest, complete); + + /* Verify middle overwrite. */ + dest = control; + memset(dest.data + 4, 0x31, 16); + compare("middle overwrite", dest, middle); + + /* Verify argument side-effects aren't repeated. */ + dest = control; + ptr = dest.data; + value = 0x60; + count = 1; + memset(ptr++, value++, count++); + ptr += 8; + memset(ptr++, value++, count++); + compare("argument side-effects", dest, three); + + /* Verify memset_after() */ + dest = control; + memset_after(&dest, 0x72, three); + compare("memset_after()", dest, after); + + /* Verify memset_startat() */ + dest = control; + memset_startat(&dest, 0x79, four); + compare("memset_startat()", dest, startat); +#undef TEST_OP +} + +static struct kunit_case memcpy_test_cases[] = { + KUNIT_CASE(memset_test), + KUNIT_CASE(memcpy_test), + KUNIT_CASE(memmove_test), + {} +}; + +static struct kunit_suite memcpy_test_suite = { + .name = "memcpy", + .test_cases = memcpy_test_cases, +}; + +kunit_test_suite(memcpy_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/lib/memregion.c b/lib/memregion.c index 77c85b5251da..be5cfa5a3b57 100644 --- a/lib/memregion.c +++ b/lib/memregion.c @@ -2,6 +2,7 @@ /* identifiers for device / performance-differentiated memory regions */ #include <linux/idr.h> #include <linux/types.h> +#include <linux/memregion.h> static DEFINE_IDA(memregion_ids); diff --git a/lib/mpi/Makefile b/lib/mpi/Makefile index d5874a7f5ff9..6e6ef9a34fe1 100644 --- a/lib/mpi/Makefile +++ b/lib/mpi/Makefile @@ -13,9 +13,16 @@ mpi-y = \ generic_mpih-rshift.o \ generic_mpih-sub1.o \ generic_mpih-add1.o \ + ec.o \ mpicoder.o \ + mpi-add.o \ mpi-bit.o \ mpi-cmp.o \ + mpi-sub-ui.o \ + mpi-div.o \ + mpi-inv.o \ + mpi-mod.o \ + mpi-mul.o \ mpih-cmp.o \ mpih-div.o \ mpih-mul.o \ diff --git a/lib/mpi/ec.c b/lib/mpi/ec.c new file mode 100644 index 000000000000..40f5908e57a4 --- /dev/null +++ b/lib/mpi/ec.c @@ -0,0 +1,1506 @@ +/* ec.c - Elliptic Curve functions + * Copyright (C) 2007 Free Software Foundation, Inc. + * Copyright (C) 2013 g10 Code GmbH + * + * This file is part of Libgcrypt. + * + * Libgcrypt is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * Libgcrypt is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "mpi-internal.h" +#include "longlong.h" + +#define point_init(a) mpi_point_init((a)) +#define point_free(a) mpi_point_free_parts((a)) + +#define log_error(fmt, ...) pr_err(fmt, ##__VA_ARGS__) +#define log_fatal(fmt, ...) pr_err(fmt, ##__VA_ARGS__) + +#define DIM(v) (sizeof(v)/sizeof((v)[0])) + + +/* Create a new point option. NBITS gives the size in bits of one + * coordinate; it is only used to pre-allocate some resources and + * might also be passed as 0 to use a default value. + */ +MPI_POINT mpi_point_new(unsigned int nbits) +{ + MPI_POINT p; + + (void)nbits; /* Currently not used. */ + + p = kmalloc(sizeof(*p), GFP_KERNEL); + if (p) + mpi_point_init(p); + return p; +} +EXPORT_SYMBOL_GPL(mpi_point_new); + +/* Release the point object P. P may be NULL. */ +void mpi_point_release(MPI_POINT p) +{ + if (p) { + mpi_point_free_parts(p); + kfree(p); + } +} +EXPORT_SYMBOL_GPL(mpi_point_release); + +/* Initialize the fields of a point object. gcry_mpi_point_free_parts + * may be used to release the fields. + */ +void mpi_point_init(MPI_POINT p) +{ + p->x = mpi_new(0); + p->y = mpi_new(0); + p->z = mpi_new(0); +} +EXPORT_SYMBOL_GPL(mpi_point_init); + +/* Release the parts of a point object. */ +void mpi_point_free_parts(MPI_POINT p) +{ + mpi_free(p->x); p->x = NULL; + mpi_free(p->y); p->y = NULL; + mpi_free(p->z); p->z = NULL; +} +EXPORT_SYMBOL_GPL(mpi_point_free_parts); + +/* Set the value from S into D. */ +static void point_set(MPI_POINT d, MPI_POINT s) +{ + mpi_set(d->x, s->x); + mpi_set(d->y, s->y); + mpi_set(d->z, s->z); +} + +static void point_resize(MPI_POINT p, struct mpi_ec_ctx *ctx) +{ + size_t nlimbs = ctx->p->nlimbs; + + mpi_resize(p->x, nlimbs); + p->x->nlimbs = nlimbs; + mpi_resize(p->z, nlimbs); + p->z->nlimbs = nlimbs; + + if (ctx->model != MPI_EC_MONTGOMERY) { + mpi_resize(p->y, nlimbs); + p->y->nlimbs = nlimbs; + } +} + +static void point_swap_cond(MPI_POINT d, MPI_POINT s, unsigned long swap, + struct mpi_ec_ctx *ctx) +{ + mpi_swap_cond(d->x, s->x, swap); + if (ctx->model != MPI_EC_MONTGOMERY) + mpi_swap_cond(d->y, s->y, swap); + mpi_swap_cond(d->z, s->z, swap); +} + + +/* W = W mod P. */ +static void ec_mod(MPI w, struct mpi_ec_ctx *ec) +{ + if (ec->t.p_barrett) + mpi_mod_barrett(w, w, ec->t.p_barrett); + else + mpi_mod(w, w, ec->p); +} + +static void ec_addm(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_add(w, u, v); + ec_mod(w, ctx); +} + +static void ec_subm(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ec) +{ + mpi_sub(w, u, v); + while (w->sign) + mpi_add(w, w, ec->p); + /*ec_mod(w, ec);*/ +} + +static void ec_mulm(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_mul(w, u, v); + ec_mod(w, ctx); +} + +/* W = 2 * U mod P. */ +static void ec_mul2(MPI w, MPI u, struct mpi_ec_ctx *ctx) +{ + mpi_lshift(w, u, 1); + ec_mod(w, ctx); +} + +static void ec_powm(MPI w, const MPI b, const MPI e, + struct mpi_ec_ctx *ctx) +{ + mpi_powm(w, b, e, ctx->p); + /* mpi_abs(w); */ +} + +/* Shortcut for + * ec_powm(B, B, mpi_const(MPI_C_TWO), ctx); + * for easier optimization. + */ +static void ec_pow2(MPI w, const MPI b, struct mpi_ec_ctx *ctx) +{ + /* Using mpi_mul is slightly faster (at least on amd64). */ + /* mpi_powm(w, b, mpi_const(MPI_C_TWO), ctx->p); */ + ec_mulm(w, b, b, ctx); +} + +/* Shortcut for + * ec_powm(B, B, mpi_const(MPI_C_THREE), ctx); + * for easier optimization. + */ +static void ec_pow3(MPI w, const MPI b, struct mpi_ec_ctx *ctx) +{ + mpi_powm(w, b, mpi_const(MPI_C_THREE), ctx->p); +} + +static void ec_invm(MPI x, MPI a, struct mpi_ec_ctx *ctx) +{ + if (!mpi_invm(x, a, ctx->p)) + log_error("ec_invm: inverse does not exist:\n"); +} + +static void mpih_set_cond(mpi_ptr_t wp, mpi_ptr_t up, + mpi_size_t usize, unsigned long set) +{ + mpi_size_t i; + mpi_limb_t mask = ((mpi_limb_t)0) - set; + mpi_limb_t x; + + for (i = 0; i < usize; i++) { + x = mask & (wp[i] ^ up[i]); + wp[i] = wp[i] ^ x; + } +} + +/* Routines for 2^255 - 19. */ + +#define LIMB_SIZE_25519 ((256+BITS_PER_MPI_LIMB-1)/BITS_PER_MPI_LIMB) + +static void ec_addm_25519(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_25519; + mpi_limb_t n[LIMB_SIZE_25519]; + mpi_limb_t borrow; + + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("addm_25519: different sizes\n"); + + memset(n, 0, sizeof(n)); + up = u->d; + vp = v->d; + wp = w->d; + + mpihelp_add_n(wp, up, vp, wsize); + borrow = mpihelp_sub_n(wp, wp, ctx->p->d, wsize); + mpih_set_cond(n, ctx->p->d, wsize, (borrow != 0UL)); + mpihelp_add_n(wp, wp, n, wsize); + wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB)); +} + +static void ec_subm_25519(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_25519; + mpi_limb_t n[LIMB_SIZE_25519]; + mpi_limb_t borrow; + + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("subm_25519: different sizes\n"); + + memset(n, 0, sizeof(n)); + up = u->d; + vp = v->d; + wp = w->d; + + borrow = mpihelp_sub_n(wp, up, vp, wsize); + mpih_set_cond(n, ctx->p->d, wsize, (borrow != 0UL)); + mpihelp_add_n(wp, wp, n, wsize); + wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB)); +} + +static void ec_mulm_25519(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_25519; + mpi_limb_t n[LIMB_SIZE_25519*2]; + mpi_limb_t m[LIMB_SIZE_25519+1]; + mpi_limb_t cy; + int msb; + + (void)ctx; + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("mulm_25519: different sizes\n"); + + up = u->d; + vp = v->d; + wp = w->d; + + mpihelp_mul_n(n, up, vp, wsize); + memcpy(wp, n, wsize * BYTES_PER_MPI_LIMB); + wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB)); + + memcpy(m, n+LIMB_SIZE_25519-1, (wsize+1) * BYTES_PER_MPI_LIMB); + mpihelp_rshift(m, m, LIMB_SIZE_25519+1, (255 % BITS_PER_MPI_LIMB)); + + memcpy(n, m, wsize * BYTES_PER_MPI_LIMB); + cy = mpihelp_lshift(m, m, LIMB_SIZE_25519, 4); + m[LIMB_SIZE_25519] = cy; + cy = mpihelp_add_n(m, m, n, wsize); + m[LIMB_SIZE_25519] += cy; + cy = mpihelp_add_n(m, m, n, wsize); + m[LIMB_SIZE_25519] += cy; + cy = mpihelp_add_n(m, m, n, wsize); + m[LIMB_SIZE_25519] += cy; + + cy = mpihelp_add_n(wp, wp, m, wsize); + m[LIMB_SIZE_25519] += cy; + + memset(m, 0, wsize * BYTES_PER_MPI_LIMB); + msb = (wp[LIMB_SIZE_25519-1] >> (255 % BITS_PER_MPI_LIMB)); + m[0] = (m[LIMB_SIZE_25519] * 2 + msb) * 19; + wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB)); + mpihelp_add_n(wp, wp, m, wsize); + + m[0] = 0; + cy = mpihelp_sub_n(wp, wp, ctx->p->d, wsize); + mpih_set_cond(m, ctx->p->d, wsize, (cy != 0UL)); + mpihelp_add_n(wp, wp, m, wsize); +} + +static void ec_mul2_25519(MPI w, MPI u, struct mpi_ec_ctx *ctx) +{ + ec_addm_25519(w, u, u, ctx); +} + +static void ec_pow2_25519(MPI w, const MPI b, struct mpi_ec_ctx *ctx) +{ + ec_mulm_25519(w, b, b, ctx); +} + +/* Routines for 2^448 - 2^224 - 1. */ + +#define LIMB_SIZE_448 ((448+BITS_PER_MPI_LIMB-1)/BITS_PER_MPI_LIMB) +#define LIMB_SIZE_HALF_448 ((LIMB_SIZE_448+1)/2) + +static void ec_addm_448(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_448; + mpi_limb_t n[LIMB_SIZE_448]; + mpi_limb_t cy; + + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("addm_448: different sizes\n"); + + memset(n, 0, sizeof(n)); + up = u->d; + vp = v->d; + wp = w->d; + + cy = mpihelp_add_n(wp, up, vp, wsize); + mpih_set_cond(n, ctx->p->d, wsize, (cy != 0UL)); + mpihelp_sub_n(wp, wp, n, wsize); +} + +static void ec_subm_448(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_448; + mpi_limb_t n[LIMB_SIZE_448]; + mpi_limb_t borrow; + + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("subm_448: different sizes\n"); + + memset(n, 0, sizeof(n)); + up = u->d; + vp = v->d; + wp = w->d; + + borrow = mpihelp_sub_n(wp, up, vp, wsize); + mpih_set_cond(n, ctx->p->d, wsize, (borrow != 0UL)); + mpihelp_add_n(wp, wp, n, wsize); +} + +static void ec_mulm_448(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_448; + mpi_limb_t n[LIMB_SIZE_448*2]; + mpi_limb_t a2[LIMB_SIZE_HALF_448]; + mpi_limb_t a3[LIMB_SIZE_HALF_448]; + mpi_limb_t b0[LIMB_SIZE_HALF_448]; + mpi_limb_t b1[LIMB_SIZE_HALF_448]; + mpi_limb_t cy; + int i; +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + mpi_limb_t b1_rest, a3_rest; +#endif + + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("mulm_448: different sizes\n"); + + up = u->d; + vp = v->d; + wp = w->d; + + mpihelp_mul_n(n, up, vp, wsize); + + for (i = 0; i < (wsize + 1) / 2; i++) { + b0[i] = n[i]; + b1[i] = n[i+wsize/2]; + a2[i] = n[i+wsize]; + a3[i] = n[i+wsize+wsize/2]; + } + +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + b0[LIMB_SIZE_HALF_448-1] &= ((mpi_limb_t)1UL << 32)-1; + a2[LIMB_SIZE_HALF_448-1] &= ((mpi_limb_t)1UL << 32)-1; + + b1_rest = 0; + a3_rest = 0; + + for (i = (wsize + 1) / 2 - 1; i >= 0; i--) { + mpi_limb_t b1v, a3v; + b1v = b1[i]; + a3v = a3[i]; + b1[i] = (b1_rest << 32) | (b1v >> 32); + a3[i] = (a3_rest << 32) | (a3v >> 32); + b1_rest = b1v & (((mpi_limb_t)1UL << 32)-1); + a3_rest = a3v & (((mpi_limb_t)1UL << 32)-1); + } +#endif + + cy = mpihelp_add_n(b0, b0, a2, LIMB_SIZE_HALF_448); + cy += mpihelp_add_n(b0, b0, a3, LIMB_SIZE_HALF_448); + for (i = 0; i < (wsize + 1) / 2; i++) + wp[i] = b0[i]; +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + wp[LIMB_SIZE_HALF_448-1] &= (((mpi_limb_t)1UL << 32)-1); +#endif + +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + cy = b0[LIMB_SIZE_HALF_448-1] >> 32; +#endif + + cy = mpihelp_add_1(b1, b1, LIMB_SIZE_HALF_448, cy); + cy += mpihelp_add_n(b1, b1, a2, LIMB_SIZE_HALF_448); + cy += mpihelp_add_n(b1, b1, a3, LIMB_SIZE_HALF_448); + cy += mpihelp_add_n(b1, b1, a3, LIMB_SIZE_HALF_448); +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + b1_rest = 0; + for (i = (wsize + 1) / 2 - 1; i >= 0; i--) { + mpi_limb_t b1v = b1[i]; + b1[i] = (b1_rest << 32) | (b1v >> 32); + b1_rest = b1v & (((mpi_limb_t)1UL << 32)-1); + } + wp[LIMB_SIZE_HALF_448-1] |= (b1_rest << 32); +#endif + for (i = 0; i < wsize / 2; i++) + wp[i+(wsize + 1) / 2] = b1[i]; + +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + cy = b1[LIMB_SIZE_HALF_448-1]; +#endif + + memset(n, 0, wsize * BYTES_PER_MPI_LIMB); + +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + n[LIMB_SIZE_HALF_448-1] = cy << 32; +#else + n[LIMB_SIZE_HALF_448] = cy; +#endif + n[0] = cy; + mpihelp_add_n(wp, wp, n, wsize); + + memset(n, 0, wsize * BYTES_PER_MPI_LIMB); + cy = mpihelp_sub_n(wp, wp, ctx->p->d, wsize); + mpih_set_cond(n, ctx->p->d, wsize, (cy != 0UL)); + mpihelp_add_n(wp, wp, n, wsize); +} + +static void ec_mul2_448(MPI w, MPI u, struct mpi_ec_ctx *ctx) +{ + ec_addm_448(w, u, u, ctx); +} + +static void ec_pow2_448(MPI w, const MPI b, struct mpi_ec_ctx *ctx) +{ + ec_mulm_448(w, b, b, ctx); +} + +struct field_table { + const char *p; + + /* computation routines for the field. */ + void (*addm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx); + void (*subm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx); + void (*mulm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx); + void (*mul2)(MPI w, MPI u, struct mpi_ec_ctx *ctx); + void (*pow2)(MPI w, const MPI b, struct mpi_ec_ctx *ctx); +}; + +static const struct field_table field_table[] = { + { + "0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFED", + ec_addm_25519, + ec_subm_25519, + ec_mulm_25519, + ec_mul2_25519, + ec_pow2_25519 + }, + { + "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE" + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + ec_addm_448, + ec_subm_448, + ec_mulm_448, + ec_mul2_448, + ec_pow2_448 + }, + { NULL, NULL, NULL, NULL, NULL, NULL }, +}; + +/* Force recomputation of all helper variables. */ +static void mpi_ec_get_reset(struct mpi_ec_ctx *ec) +{ + ec->t.valid.a_is_pminus3 = 0; + ec->t.valid.two_inv_p = 0; +} + +/* Accessor for helper variable. */ +static int ec_get_a_is_pminus3(struct mpi_ec_ctx *ec) +{ + MPI tmp; + + if (!ec->t.valid.a_is_pminus3) { + ec->t.valid.a_is_pminus3 = 1; + tmp = mpi_alloc_like(ec->p); + mpi_sub_ui(tmp, ec->p, 3); + ec->t.a_is_pminus3 = !mpi_cmp(ec->a, tmp); + mpi_free(tmp); + } + + return ec->t.a_is_pminus3; +} + +/* Accessor for helper variable. */ +static MPI ec_get_two_inv_p(struct mpi_ec_ctx *ec) +{ + if (!ec->t.valid.two_inv_p) { + ec->t.valid.two_inv_p = 1; + if (!ec->t.two_inv_p) + ec->t.two_inv_p = mpi_alloc(0); + ec_invm(ec->t.two_inv_p, mpi_const(MPI_C_TWO), ec); + } + return ec->t.two_inv_p; +} + +static const char *const curve25519_bad_points[] = { + "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed", + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000000000000000000000000000001", + "0x00b8495f16056286fdb1329ceb8d09da6ac49ff1fae35616aeb8413b7c7aebe0", + "0x57119fd0dd4e22d8868e1c58c45c44045bef839c55b1d0b1248c50a3bc959c5f", + "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffec", + "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffee", + NULL +}; + +static const char *const curve448_bad_points[] = { + "0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffe" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "0x00000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000", + "0x00000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000001", + "0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffe" + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffe", + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "00000000000000000000000000000000000000000000000000000000", + NULL +}; + +static const char *const *bad_points_table[] = { + curve25519_bad_points, + curve448_bad_points, +}; + +static void mpi_ec_coefficient_normalize(MPI a, MPI p) +{ + if (a->sign) { + mpi_resize(a, p->nlimbs); + mpihelp_sub_n(a->d, p->d, a->d, p->nlimbs); + a->nlimbs = p->nlimbs; + a->sign = 0; + } +} + +/* This function initialized a context for elliptic curve based on the + * field GF(p). P is the prime specifying this field, A is the first + * coefficient. CTX is expected to be zeroized. + */ +void mpi_ec_init(struct mpi_ec_ctx *ctx, enum gcry_mpi_ec_models model, + enum ecc_dialects dialect, + int flags, MPI p, MPI a, MPI b) +{ + int i; + static int use_barrett = -1 /* TODO: 1 or -1 */; + + mpi_ec_coefficient_normalize(a, p); + mpi_ec_coefficient_normalize(b, p); + + /* Fixme: Do we want to check some constraints? e.g. a < p */ + + ctx->model = model; + ctx->dialect = dialect; + ctx->flags = flags; + if (dialect == ECC_DIALECT_ED25519) + ctx->nbits = 256; + else + ctx->nbits = mpi_get_nbits(p); + ctx->p = mpi_copy(p); + ctx->a = mpi_copy(a); + ctx->b = mpi_copy(b); + + ctx->t.p_barrett = use_barrett > 0 ? mpi_barrett_init(ctx->p, 0) : NULL; + + mpi_ec_get_reset(ctx); + + if (model == MPI_EC_MONTGOMERY) { + for (i = 0; i < DIM(bad_points_table); i++) { + MPI p_candidate = mpi_scanval(bad_points_table[i][0]); + int match_p = !mpi_cmp(ctx->p, p_candidate); + int j; + + mpi_free(p_candidate); + if (!match_p) + continue; + + for (j = 0; i < DIM(ctx->t.scratch) && bad_points_table[i][j]; j++) + ctx->t.scratch[j] = mpi_scanval(bad_points_table[i][j]); + } + } else { + /* Allocate scratch variables. */ + for (i = 0; i < DIM(ctx->t.scratch); i++) + ctx->t.scratch[i] = mpi_alloc_like(ctx->p); + } + + ctx->addm = ec_addm; + ctx->subm = ec_subm; + ctx->mulm = ec_mulm; + ctx->mul2 = ec_mul2; + ctx->pow2 = ec_pow2; + + for (i = 0; field_table[i].p; i++) { + MPI f_p; + + f_p = mpi_scanval(field_table[i].p); + if (!f_p) + break; + + if (!mpi_cmp(p, f_p)) { + ctx->addm = field_table[i].addm; + ctx->subm = field_table[i].subm; + ctx->mulm = field_table[i].mulm; + ctx->mul2 = field_table[i].mul2; + ctx->pow2 = field_table[i].pow2; + mpi_free(f_p); + + mpi_resize(ctx->a, ctx->p->nlimbs); + ctx->a->nlimbs = ctx->p->nlimbs; + + mpi_resize(ctx->b, ctx->p->nlimbs); + ctx->b->nlimbs = ctx->p->nlimbs; + + for (i = 0; i < DIM(ctx->t.scratch) && ctx->t.scratch[i]; i++) + ctx->t.scratch[i]->nlimbs = ctx->p->nlimbs; + + break; + } + + mpi_free(f_p); + } +} +EXPORT_SYMBOL_GPL(mpi_ec_init); + +void mpi_ec_deinit(struct mpi_ec_ctx *ctx) +{ + int i; + + mpi_barrett_free(ctx->t.p_barrett); + + /* Domain parameter. */ + mpi_free(ctx->p); + mpi_free(ctx->a); + mpi_free(ctx->b); + mpi_point_release(ctx->G); + mpi_free(ctx->n); + + /* The key. */ + mpi_point_release(ctx->Q); + mpi_free(ctx->d); + + /* Private data of ec.c. */ + mpi_free(ctx->t.two_inv_p); + + for (i = 0; i < DIM(ctx->t.scratch); i++) + mpi_free(ctx->t.scratch[i]); +} +EXPORT_SYMBOL_GPL(mpi_ec_deinit); + +/* Compute the affine coordinates from the projective coordinates in + * POINT. Set them into X and Y. If one coordinate is not required, + * X or Y may be passed as NULL. CTX is the usual context. Returns: 0 + * on success or !0 if POINT is at infinity. + */ +int mpi_ec_get_affine(MPI x, MPI y, MPI_POINT point, struct mpi_ec_ctx *ctx) +{ + if (!mpi_cmp_ui(point->z, 0)) + return -1; + + switch (ctx->model) { + case MPI_EC_WEIERSTRASS: /* Using Jacobian coordinates. */ + { + MPI z1, z2, z3; + + z1 = mpi_new(0); + z2 = mpi_new(0); + ec_invm(z1, point->z, ctx); /* z1 = z^(-1) mod p */ + ec_mulm(z2, z1, z1, ctx); /* z2 = z^(-2) mod p */ + + if (x) + ec_mulm(x, point->x, z2, ctx); + + if (y) { + z3 = mpi_new(0); + ec_mulm(z3, z2, z1, ctx); /* z3 = z^(-3) mod p */ + ec_mulm(y, point->y, z3, ctx); + mpi_free(z3); + } + + mpi_free(z2); + mpi_free(z1); + } + return 0; + + case MPI_EC_MONTGOMERY: + { + if (x) + mpi_set(x, point->x); + + if (y) { + log_fatal("%s: Getting Y-coordinate on %s is not supported\n", + "mpi_ec_get_affine", "Montgomery"); + return -1; + } + } + return 0; + + case MPI_EC_EDWARDS: + { + MPI z; + + z = mpi_new(0); + ec_invm(z, point->z, ctx); + + mpi_resize(z, ctx->p->nlimbs); + z->nlimbs = ctx->p->nlimbs; + + if (x) { + mpi_resize(x, ctx->p->nlimbs); + x->nlimbs = ctx->p->nlimbs; + ctx->mulm(x, point->x, z, ctx); + } + if (y) { + mpi_resize(y, ctx->p->nlimbs); + y->nlimbs = ctx->p->nlimbs; + ctx->mulm(y, point->y, z, ctx); + } + + mpi_free(z); + } + return 0; + + default: + return -1; + } +} +EXPORT_SYMBOL_GPL(mpi_ec_get_affine); + +/* RESULT = 2 * POINT (Weierstrass version). */ +static void dup_point_weierstrass(MPI_POINT result, + MPI_POINT point, struct mpi_ec_ctx *ctx) +{ +#define x3 (result->x) +#define y3 (result->y) +#define z3 (result->z) +#define t1 (ctx->t.scratch[0]) +#define t2 (ctx->t.scratch[1]) +#define t3 (ctx->t.scratch[2]) +#define l1 (ctx->t.scratch[3]) +#define l2 (ctx->t.scratch[4]) +#define l3 (ctx->t.scratch[5]) + + if (!mpi_cmp_ui(point->y, 0) || !mpi_cmp_ui(point->z, 0)) { + /* P_y == 0 || P_z == 0 => [1:1:0] */ + mpi_set_ui(x3, 1); + mpi_set_ui(y3, 1); + mpi_set_ui(z3, 0); + } else { + if (ec_get_a_is_pminus3(ctx)) { + /* Use the faster case. */ + /* L1 = 3(X - Z^2)(X + Z^2) */ + /* T1: used for Z^2. */ + /* T2: used for the right term. */ + ec_pow2(t1, point->z, ctx); + ec_subm(l1, point->x, t1, ctx); + ec_mulm(l1, l1, mpi_const(MPI_C_THREE), ctx); + ec_addm(t2, point->x, t1, ctx); + ec_mulm(l1, l1, t2, ctx); + } else { + /* Standard case. */ + /* L1 = 3X^2 + aZ^4 */ + /* T1: used for aZ^4. */ + ec_pow2(l1, point->x, ctx); + ec_mulm(l1, l1, mpi_const(MPI_C_THREE), ctx); + ec_powm(t1, point->z, mpi_const(MPI_C_FOUR), ctx); + ec_mulm(t1, t1, ctx->a, ctx); + ec_addm(l1, l1, t1, ctx); + } + /* Z3 = 2YZ */ + ec_mulm(z3, point->y, point->z, ctx); + ec_mul2(z3, z3, ctx); + + /* L2 = 4XY^2 */ + /* T2: used for Y2; required later. */ + ec_pow2(t2, point->y, ctx); + ec_mulm(l2, t2, point->x, ctx); + ec_mulm(l2, l2, mpi_const(MPI_C_FOUR), ctx); + + /* X3 = L1^2 - 2L2 */ + /* T1: used for L2^2. */ + ec_pow2(x3, l1, ctx); + ec_mul2(t1, l2, ctx); + ec_subm(x3, x3, t1, ctx); + + /* L3 = 8Y^4 */ + /* T2: taken from above. */ + ec_pow2(t2, t2, ctx); + ec_mulm(l3, t2, mpi_const(MPI_C_EIGHT), ctx); + + /* Y3 = L1(L2 - X3) - L3 */ + ec_subm(y3, l2, x3, ctx); + ec_mulm(y3, y3, l1, ctx); + ec_subm(y3, y3, l3, ctx); + } + +#undef x3 +#undef y3 +#undef z3 +#undef t1 +#undef t2 +#undef t3 +#undef l1 +#undef l2 +#undef l3 +} + +/* RESULT = 2 * POINT (Montgomery version). */ +static void dup_point_montgomery(MPI_POINT result, + MPI_POINT point, struct mpi_ec_ctx *ctx) +{ + (void)result; + (void)point; + (void)ctx; + log_fatal("%s: %s not yet supported\n", + "mpi_ec_dup_point", "Montgomery"); +} + +/* RESULT = 2 * POINT (Twisted Edwards version). */ +static void dup_point_edwards(MPI_POINT result, + MPI_POINT point, struct mpi_ec_ctx *ctx) +{ +#define X1 (point->x) +#define Y1 (point->y) +#define Z1 (point->z) +#define X3 (result->x) +#define Y3 (result->y) +#define Z3 (result->z) +#define B (ctx->t.scratch[0]) +#define C (ctx->t.scratch[1]) +#define D (ctx->t.scratch[2]) +#define E (ctx->t.scratch[3]) +#define F (ctx->t.scratch[4]) +#define H (ctx->t.scratch[5]) +#define J (ctx->t.scratch[6]) + + /* Compute: (X_3 : Y_3 : Z_3) = 2( X_1 : Y_1 : Z_1 ) */ + + /* B = (X_1 + Y_1)^2 */ + ctx->addm(B, X1, Y1, ctx); + ctx->pow2(B, B, ctx); + + /* C = X_1^2 */ + /* D = Y_1^2 */ + ctx->pow2(C, X1, ctx); + ctx->pow2(D, Y1, ctx); + + /* E = aC */ + if (ctx->dialect == ECC_DIALECT_ED25519) + ctx->subm(E, ctx->p, C, ctx); + else + ctx->mulm(E, ctx->a, C, ctx); + + /* F = E + D */ + ctx->addm(F, E, D, ctx); + + /* H = Z_1^2 */ + ctx->pow2(H, Z1, ctx); + + /* J = F - 2H */ + ctx->mul2(J, H, ctx); + ctx->subm(J, F, J, ctx); + + /* X_3 = (B - C - D) · J */ + ctx->subm(X3, B, C, ctx); + ctx->subm(X3, X3, D, ctx); + ctx->mulm(X3, X3, J, ctx); + + /* Y_3 = F · (E - D) */ + ctx->subm(Y3, E, D, ctx); + ctx->mulm(Y3, Y3, F, ctx); + + /* Z_3 = F · J */ + ctx->mulm(Z3, F, J, ctx); + +#undef X1 +#undef Y1 +#undef Z1 +#undef X3 +#undef Y3 +#undef Z3 +#undef B +#undef C +#undef D +#undef E +#undef F +#undef H +#undef J +} + +/* RESULT = 2 * POINT */ +static void +mpi_ec_dup_point(MPI_POINT result, MPI_POINT point, struct mpi_ec_ctx *ctx) +{ + switch (ctx->model) { + case MPI_EC_WEIERSTRASS: + dup_point_weierstrass(result, point, ctx); + break; + case MPI_EC_MONTGOMERY: + dup_point_montgomery(result, point, ctx); + break; + case MPI_EC_EDWARDS: + dup_point_edwards(result, point, ctx); + break; + } +} + +/* RESULT = P1 + P2 (Weierstrass version).*/ +static void add_points_weierstrass(MPI_POINT result, + MPI_POINT p1, MPI_POINT p2, + struct mpi_ec_ctx *ctx) +{ +#define x1 (p1->x) +#define y1 (p1->y) +#define z1 (p1->z) +#define x2 (p2->x) +#define y2 (p2->y) +#define z2 (p2->z) +#define x3 (result->x) +#define y3 (result->y) +#define z3 (result->z) +#define l1 (ctx->t.scratch[0]) +#define l2 (ctx->t.scratch[1]) +#define l3 (ctx->t.scratch[2]) +#define l4 (ctx->t.scratch[3]) +#define l5 (ctx->t.scratch[4]) +#define l6 (ctx->t.scratch[5]) +#define l7 (ctx->t.scratch[6]) +#define l8 (ctx->t.scratch[7]) +#define l9 (ctx->t.scratch[8]) +#define t1 (ctx->t.scratch[9]) +#define t2 (ctx->t.scratch[10]) + + if ((!mpi_cmp(x1, x2)) && (!mpi_cmp(y1, y2)) && (!mpi_cmp(z1, z2))) { + /* Same point; need to call the duplicate function. */ + mpi_ec_dup_point(result, p1, ctx); + } else if (!mpi_cmp_ui(z1, 0)) { + /* P1 is at infinity. */ + mpi_set(x3, p2->x); + mpi_set(y3, p2->y); + mpi_set(z3, p2->z); + } else if (!mpi_cmp_ui(z2, 0)) { + /* P2 is at infinity. */ + mpi_set(x3, p1->x); + mpi_set(y3, p1->y); + mpi_set(z3, p1->z); + } else { + int z1_is_one = !mpi_cmp_ui(z1, 1); + int z2_is_one = !mpi_cmp_ui(z2, 1); + + /* l1 = x1 z2^2 */ + /* l2 = x2 z1^2 */ + if (z2_is_one) + mpi_set(l1, x1); + else { + ec_pow2(l1, z2, ctx); + ec_mulm(l1, l1, x1, ctx); + } + if (z1_is_one) + mpi_set(l2, x2); + else { + ec_pow2(l2, z1, ctx); + ec_mulm(l2, l2, x2, ctx); + } + /* l3 = l1 - l2 */ + ec_subm(l3, l1, l2, ctx); + /* l4 = y1 z2^3 */ + ec_powm(l4, z2, mpi_const(MPI_C_THREE), ctx); + ec_mulm(l4, l4, y1, ctx); + /* l5 = y2 z1^3 */ + ec_powm(l5, z1, mpi_const(MPI_C_THREE), ctx); + ec_mulm(l5, l5, y2, ctx); + /* l6 = l4 - l5 */ + ec_subm(l6, l4, l5, ctx); + + if (!mpi_cmp_ui(l3, 0)) { + if (!mpi_cmp_ui(l6, 0)) { + /* P1 and P2 are the same - use duplicate function. */ + mpi_ec_dup_point(result, p1, ctx); + } else { + /* P1 is the inverse of P2. */ + mpi_set_ui(x3, 1); + mpi_set_ui(y3, 1); + mpi_set_ui(z3, 0); + } + } else { + /* l7 = l1 + l2 */ + ec_addm(l7, l1, l2, ctx); + /* l8 = l4 + l5 */ + ec_addm(l8, l4, l5, ctx); + /* z3 = z1 z2 l3 */ + ec_mulm(z3, z1, z2, ctx); + ec_mulm(z3, z3, l3, ctx); + /* x3 = l6^2 - l7 l3^2 */ + ec_pow2(t1, l6, ctx); + ec_pow2(t2, l3, ctx); + ec_mulm(t2, t2, l7, ctx); + ec_subm(x3, t1, t2, ctx); + /* l9 = l7 l3^2 - 2 x3 */ + ec_mul2(t1, x3, ctx); + ec_subm(l9, t2, t1, ctx); + /* y3 = (l9 l6 - l8 l3^3)/2 */ + ec_mulm(l9, l9, l6, ctx); + ec_powm(t1, l3, mpi_const(MPI_C_THREE), ctx); /* fixme: Use saved value*/ + ec_mulm(t1, t1, l8, ctx); + ec_subm(y3, l9, t1, ctx); + ec_mulm(y3, y3, ec_get_two_inv_p(ctx), ctx); + } + } + +#undef x1 +#undef y1 +#undef z1 +#undef x2 +#undef y2 +#undef z2 +#undef x3 +#undef y3 +#undef z3 +#undef l1 +#undef l2 +#undef l3 +#undef l4 +#undef l5 +#undef l6 +#undef l7 +#undef l8 +#undef l9 +#undef t1 +#undef t2 +} + +/* RESULT = P1 + P2 (Montgomery version).*/ +static void add_points_montgomery(MPI_POINT result, + MPI_POINT p1, MPI_POINT p2, + struct mpi_ec_ctx *ctx) +{ + (void)result; + (void)p1; + (void)p2; + (void)ctx; + log_fatal("%s: %s not yet supported\n", + "mpi_ec_add_points", "Montgomery"); +} + +/* RESULT = P1 + P2 (Twisted Edwards version).*/ +static void add_points_edwards(MPI_POINT result, + MPI_POINT p1, MPI_POINT p2, + struct mpi_ec_ctx *ctx) +{ +#define X1 (p1->x) +#define Y1 (p1->y) +#define Z1 (p1->z) +#define X2 (p2->x) +#define Y2 (p2->y) +#define Z2 (p2->z) +#define X3 (result->x) +#define Y3 (result->y) +#define Z3 (result->z) +#define A (ctx->t.scratch[0]) +#define B (ctx->t.scratch[1]) +#define C (ctx->t.scratch[2]) +#define D (ctx->t.scratch[3]) +#define E (ctx->t.scratch[4]) +#define F (ctx->t.scratch[5]) +#define G (ctx->t.scratch[6]) +#define tmp (ctx->t.scratch[7]) + + point_resize(result, ctx); + + /* Compute: (X_3 : Y_3 : Z_3) = (X_1 : Y_1 : Z_1) + (X_2 : Y_2 : Z_3) */ + + /* A = Z1 · Z2 */ + ctx->mulm(A, Z1, Z2, ctx); + + /* B = A^2 */ + ctx->pow2(B, A, ctx); + + /* C = X1 · X2 */ + ctx->mulm(C, X1, X2, ctx); + + /* D = Y1 · Y2 */ + ctx->mulm(D, Y1, Y2, ctx); + + /* E = d · C · D */ + ctx->mulm(E, ctx->b, C, ctx); + ctx->mulm(E, E, D, ctx); + + /* F = B - E */ + ctx->subm(F, B, E, ctx); + + /* G = B + E */ + ctx->addm(G, B, E, ctx); + + /* X_3 = A · F · ((X_1 + Y_1) · (X_2 + Y_2) - C - D) */ + ctx->addm(tmp, X1, Y1, ctx); + ctx->addm(X3, X2, Y2, ctx); + ctx->mulm(X3, X3, tmp, ctx); + ctx->subm(X3, X3, C, ctx); + ctx->subm(X3, X3, D, ctx); + ctx->mulm(X3, X3, F, ctx); + ctx->mulm(X3, X3, A, ctx); + + /* Y_3 = A · G · (D - aC) */ + if (ctx->dialect == ECC_DIALECT_ED25519) { + ctx->addm(Y3, D, C, ctx); + } else { + ctx->mulm(Y3, ctx->a, C, ctx); + ctx->subm(Y3, D, Y3, ctx); + } + ctx->mulm(Y3, Y3, G, ctx); + ctx->mulm(Y3, Y3, A, ctx); + + /* Z_3 = F · G */ + ctx->mulm(Z3, F, G, ctx); + + +#undef X1 +#undef Y1 +#undef Z1 +#undef X2 +#undef Y2 +#undef Z2 +#undef X3 +#undef Y3 +#undef Z3 +#undef A +#undef B +#undef C +#undef D +#undef E +#undef F +#undef G +#undef tmp +} + +/* Compute a step of Montgomery Ladder (only use X and Z in the point). + * Inputs: P1, P2, and x-coordinate of DIF = P1 - P1. + * Outputs: PRD = 2 * P1 and SUM = P1 + P2. + */ +static void montgomery_ladder(MPI_POINT prd, MPI_POINT sum, + MPI_POINT p1, MPI_POINT p2, MPI dif_x, + struct mpi_ec_ctx *ctx) +{ + ctx->addm(sum->x, p2->x, p2->z, ctx); + ctx->subm(p2->z, p2->x, p2->z, ctx); + ctx->addm(prd->x, p1->x, p1->z, ctx); + ctx->subm(p1->z, p1->x, p1->z, ctx); + ctx->mulm(p2->x, p1->z, sum->x, ctx); + ctx->mulm(p2->z, prd->x, p2->z, ctx); + ctx->pow2(p1->x, prd->x, ctx); + ctx->pow2(p1->z, p1->z, ctx); + ctx->addm(sum->x, p2->x, p2->z, ctx); + ctx->subm(p2->z, p2->x, p2->z, ctx); + ctx->mulm(prd->x, p1->x, p1->z, ctx); + ctx->subm(p1->z, p1->x, p1->z, ctx); + ctx->pow2(sum->x, sum->x, ctx); + ctx->pow2(sum->z, p2->z, ctx); + ctx->mulm(prd->z, p1->z, ctx->a, ctx); /* CTX->A: (a-2)/4 */ + ctx->mulm(sum->z, sum->z, dif_x, ctx); + ctx->addm(prd->z, p1->x, prd->z, ctx); + ctx->mulm(prd->z, prd->z, p1->z, ctx); +} + +/* RESULT = P1 + P2 */ +void mpi_ec_add_points(MPI_POINT result, + MPI_POINT p1, MPI_POINT p2, + struct mpi_ec_ctx *ctx) +{ + switch (ctx->model) { + case MPI_EC_WEIERSTRASS: + add_points_weierstrass(result, p1, p2, ctx); + break; + case MPI_EC_MONTGOMERY: + add_points_montgomery(result, p1, p2, ctx); + break; + case MPI_EC_EDWARDS: + add_points_edwards(result, p1, p2, ctx); + break; + } +} +EXPORT_SYMBOL_GPL(mpi_ec_add_points); + +/* Scalar point multiplication - the main function for ECC. If takes + * an integer SCALAR and a POINT as well as the usual context CTX. + * RESULT will be set to the resulting point. + */ +void mpi_ec_mul_point(MPI_POINT result, + MPI scalar, MPI_POINT point, + struct mpi_ec_ctx *ctx) +{ + MPI x1, y1, z1, k, h, yy; + unsigned int i, loops; + struct gcry_mpi_point p1, p2, p1inv; + + if (ctx->model == MPI_EC_EDWARDS) { + /* Simple left to right binary method. Algorithm 3.27 from + * {author={Hankerson, Darrel and Menezes, Alfred J. and Vanstone, Scott}, + * title = {Guide to Elliptic Curve Cryptography}, + * year = {2003}, isbn = {038795273X}, + * url = {http://www.cacr.math.uwaterloo.ca/ecc/}, + * publisher = {Springer-Verlag New York, Inc.}} + */ + unsigned int nbits; + int j; + + if (mpi_cmp(scalar, ctx->p) >= 0) + nbits = mpi_get_nbits(scalar); + else + nbits = mpi_get_nbits(ctx->p); + + mpi_set_ui(result->x, 0); + mpi_set_ui(result->y, 1); + mpi_set_ui(result->z, 1); + point_resize(point, ctx); + + point_resize(result, ctx); + point_resize(point, ctx); + + for (j = nbits-1; j >= 0; j--) { + mpi_ec_dup_point(result, result, ctx); + if (mpi_test_bit(scalar, j)) + mpi_ec_add_points(result, result, point, ctx); + } + return; + } else if (ctx->model == MPI_EC_MONTGOMERY) { + unsigned int nbits; + int j; + struct gcry_mpi_point p1_, p2_; + MPI_POINT q1, q2, prd, sum; + unsigned long sw; + mpi_size_t rsize; + + /* Compute scalar point multiplication with Montgomery Ladder. + * Note that we don't use Y-coordinate in the points at all. + * RESULT->Y will be filled by zero. + */ + + nbits = mpi_get_nbits(scalar); + point_init(&p1); + point_init(&p2); + point_init(&p1_); + point_init(&p2_); + mpi_set_ui(p1.x, 1); + mpi_free(p2.x); + p2.x = mpi_copy(point->x); + mpi_set_ui(p2.z, 1); + + point_resize(&p1, ctx); + point_resize(&p2, ctx); + point_resize(&p1_, ctx); + point_resize(&p2_, ctx); + + mpi_resize(point->x, ctx->p->nlimbs); + point->x->nlimbs = ctx->p->nlimbs; + + q1 = &p1; + q2 = &p2; + prd = &p1_; + sum = &p2_; + + for (j = nbits-1; j >= 0; j--) { + MPI_POINT t; + + sw = mpi_test_bit(scalar, j); + point_swap_cond(q1, q2, sw, ctx); + montgomery_ladder(prd, sum, q1, q2, point->x, ctx); + point_swap_cond(prd, sum, sw, ctx); + t = q1; q1 = prd; prd = t; + t = q2; q2 = sum; sum = t; + } + + mpi_clear(result->y); + sw = (nbits & 1); + point_swap_cond(&p1, &p1_, sw, ctx); + + rsize = p1.z->nlimbs; + MPN_NORMALIZE(p1.z->d, rsize); + if (rsize == 0) { + mpi_set_ui(result->x, 1); + mpi_set_ui(result->z, 0); + } else { + z1 = mpi_new(0); + ec_invm(z1, p1.z, ctx); + ec_mulm(result->x, p1.x, z1, ctx); + mpi_set_ui(result->z, 1); + mpi_free(z1); + } + + point_free(&p1); + point_free(&p2); + point_free(&p1_); + point_free(&p2_); + return; + } + + x1 = mpi_alloc_like(ctx->p); + y1 = mpi_alloc_like(ctx->p); + h = mpi_alloc_like(ctx->p); + k = mpi_copy(scalar); + yy = mpi_copy(point->y); + + if (mpi_has_sign(k)) { + k->sign = 0; + ec_invm(yy, yy, ctx); + } + + if (!mpi_cmp_ui(point->z, 1)) { + mpi_set(x1, point->x); + mpi_set(y1, yy); + } else { + MPI z2, z3; + + z2 = mpi_alloc_like(ctx->p); + z3 = mpi_alloc_like(ctx->p); + ec_mulm(z2, point->z, point->z, ctx); + ec_mulm(z3, point->z, z2, ctx); + ec_invm(z2, z2, ctx); + ec_mulm(x1, point->x, z2, ctx); + ec_invm(z3, z3, ctx); + ec_mulm(y1, yy, z3, ctx); + mpi_free(z2); + mpi_free(z3); + } + z1 = mpi_copy(mpi_const(MPI_C_ONE)); + + mpi_mul(h, k, mpi_const(MPI_C_THREE)); /* h = 3k */ + loops = mpi_get_nbits(h); + if (loops < 2) { + /* If SCALAR is zero, the above mpi_mul sets H to zero and thus + * LOOPs will be zero. To avoid an underflow of I in the main + * loop we set LOOP to 2 and the result to (0,0,0). + */ + loops = 2; + mpi_clear(result->x); + mpi_clear(result->y); + mpi_clear(result->z); + } else { + mpi_set(result->x, point->x); + mpi_set(result->y, yy); + mpi_set(result->z, point->z); + } + mpi_free(yy); yy = NULL; + + p1.x = x1; x1 = NULL; + p1.y = y1; y1 = NULL; + p1.z = z1; z1 = NULL; + point_init(&p2); + point_init(&p1inv); + + /* Invert point: y = p - y mod p */ + point_set(&p1inv, &p1); + ec_subm(p1inv.y, ctx->p, p1inv.y, ctx); + + for (i = loops-2; i > 0; i--) { + mpi_ec_dup_point(result, result, ctx); + if (mpi_test_bit(h, i) == 1 && mpi_test_bit(k, i) == 0) { + point_set(&p2, result); + mpi_ec_add_points(result, &p2, &p1, ctx); + } + if (mpi_test_bit(h, i) == 0 && mpi_test_bit(k, i) == 1) { + point_set(&p2, result); + mpi_ec_add_points(result, &p2, &p1inv, ctx); + } + } + + point_free(&p1); + point_free(&p2); + point_free(&p1inv); + mpi_free(h); + mpi_free(k); +} +EXPORT_SYMBOL_GPL(mpi_ec_mul_point); + +/* Return true if POINT is on the curve described by CTX. */ +int mpi_ec_curve_point(MPI_POINT point, struct mpi_ec_ctx *ctx) +{ + int res = 0; + MPI x, y, w; + + x = mpi_new(0); + y = mpi_new(0); + w = mpi_new(0); + + /* Check that the point is in range. This needs to be done here and + * not after conversion to affine coordinates. + */ + if (mpi_cmpabs(point->x, ctx->p) >= 0) + goto leave; + if (mpi_cmpabs(point->y, ctx->p) >= 0) + goto leave; + if (mpi_cmpabs(point->z, ctx->p) >= 0) + goto leave; + + switch (ctx->model) { + case MPI_EC_WEIERSTRASS: + { + MPI xxx; + + if (mpi_ec_get_affine(x, y, point, ctx)) + goto leave; + + xxx = mpi_new(0); + + /* y^2 == x^3 + a·x + b */ + ec_pow2(y, y, ctx); + + ec_pow3(xxx, x, ctx); + ec_mulm(w, ctx->a, x, ctx); + ec_addm(w, w, ctx->b, ctx); + ec_addm(w, w, xxx, ctx); + + if (!mpi_cmp(y, w)) + res = 1; + + mpi_free(xxx); + } + break; + + case MPI_EC_MONTGOMERY: + { +#define xx y + /* With Montgomery curve, only X-coordinate is valid. */ + if (mpi_ec_get_affine(x, NULL, point, ctx)) + goto leave; + + /* The equation is: b * y^2 == x^3 + a · x^2 + x */ + /* We check if right hand is quadratic residue or not by + * Euler's criterion. + */ + /* CTX->A has (a-2)/4 and CTX->B has b^-1 */ + ec_mulm(w, ctx->a, mpi_const(MPI_C_FOUR), ctx); + ec_addm(w, w, mpi_const(MPI_C_TWO), ctx); + ec_mulm(w, w, x, ctx); + ec_pow2(xx, x, ctx); + ec_addm(w, w, xx, ctx); + ec_addm(w, w, mpi_const(MPI_C_ONE), ctx); + ec_mulm(w, w, x, ctx); + ec_mulm(w, w, ctx->b, ctx); +#undef xx + /* Compute Euler's criterion: w^(p-1)/2 */ +#define p_minus1 y + ec_subm(p_minus1, ctx->p, mpi_const(MPI_C_ONE), ctx); + mpi_rshift(p_minus1, p_minus1, 1); + ec_powm(w, w, p_minus1, ctx); + + res = !mpi_cmp_ui(w, 1); +#undef p_minus1 + } + break; + + case MPI_EC_EDWARDS: + { + if (mpi_ec_get_affine(x, y, point, ctx)) + goto leave; + + mpi_resize(w, ctx->p->nlimbs); + w->nlimbs = ctx->p->nlimbs; + + /* a · x^2 + y^2 - 1 - b · x^2 · y^2 == 0 */ + ctx->pow2(x, x, ctx); + ctx->pow2(y, y, ctx); + if (ctx->dialect == ECC_DIALECT_ED25519) + ctx->subm(w, ctx->p, x, ctx); + else + ctx->mulm(w, ctx->a, x, ctx); + ctx->addm(w, w, y, ctx); + ctx->mulm(x, x, y, ctx); + ctx->mulm(x, x, ctx->b, ctx); + ctx->subm(w, w, x, ctx); + if (!mpi_cmp_ui(w, 1)) + res = 1; + } + break; + } + +leave: + mpi_free(w); + mpi_free(x); + mpi_free(y); + + return res; +} +EXPORT_SYMBOL_GPL(mpi_ec_curve_point); diff --git a/lib/mpi/longlong.h b/lib/mpi/longlong.h index 2dceaca27489..b6fa1d08fb55 100644 --- a/lib/mpi/longlong.h +++ b/lib/mpi/longlong.h @@ -48,8 +48,8 @@ /* Define auxiliary asm macros. * - * 1) umul_ppmm(high_prod, low_prod, multipler, multiplicand) multiplies two - * UWtype integers MULTIPLER and MULTIPLICAND, and generates a two UWtype + * 1) umul_ppmm(high_prod, low_prod, multiplier, multiplicand) multiplies two + * UWtype integers MULTIPLIER and MULTIPLICAND, and generates a two UWtype * word product in HIGH_PROD and LOW_PROD. * * 2) __umulsidi3(a,b) multiplies two UWtype integers A and B, and returns a @@ -653,7 +653,7 @@ do { \ ************** MIPS/64 ************** ***************************************/ #if (defined(__mips) && __mips >= 3) && W_TYPE_SIZE == 64 -#if defined(__mips_isa_rev) && __mips_isa_rev >= 6 +#if defined(__mips_isa_rev) && __mips_isa_rev >= 6 && defined(CONFIG_CC_IS_GCC) /* * GCC ends up emitting a __multi3 intrinsic call for MIPS64r6 with the plain C * code below, so we special case MIPS64r6 until the compiler can do better. @@ -722,22 +722,22 @@ do { \ do { \ if (__builtin_constant_p(bh) && (bh) == 0) \ __asm__ ("{a%I4|add%I4c} %1,%3,%4\n\t{aze|addze} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "%r" ((USItype)(ah)), \ "%r" ((USItype)(al)), \ "rI" ((USItype)(bl))); \ else if (__builtin_constant_p(bh) && (bh) == ~(USItype) 0) \ __asm__ ("{a%I4|add%I4c} %1,%3,%4\n\t{ame|addme} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "%r" ((USItype)(ah)), \ "%r" ((USItype)(al)), \ "rI" ((USItype)(bl))); \ else \ __asm__ ("{a%I5|add%I5c} %1,%4,%5\n\t{ae|adde} %0,%2,%3" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "%r" ((USItype)(ah)), \ "r" ((USItype)(bh)), \ "%r" ((USItype)(al)), \ @@ -747,36 +747,36 @@ do { \ do { \ if (__builtin_constant_p(ah) && (ah) == 0) \ __asm__ ("{sf%I3|subf%I3c} %1,%4,%3\n\t{sfze|subfze} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "r" ((USItype)(bh)), \ "rI" ((USItype)(al)), \ "r" ((USItype)(bl))); \ else if (__builtin_constant_p(ah) && (ah) == ~(USItype) 0) \ __asm__ ("{sf%I3|subf%I3c} %1,%4,%3\n\t{sfme|subfme} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "r" ((USItype)(bh)), \ "rI" ((USItype)(al)), \ "r" ((USItype)(bl))); \ else if (__builtin_constant_p(bh) && (bh) == 0) \ __asm__ ("{sf%I3|subf%I3c} %1,%4,%3\n\t{ame|addme} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "r" ((USItype)(ah)), \ "rI" ((USItype)(al)), \ "r" ((USItype)(bl))); \ else if (__builtin_constant_p(bh) && (bh) == ~(USItype) 0) \ __asm__ ("{sf%I3|subf%I3c} %1,%4,%3\n\t{aze|addze} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "r" ((USItype)(ah)), \ "rI" ((USItype)(al)), \ "r" ((USItype)(bl))); \ else \ __asm__ ("{sf%I4|subf%I4c} %1,%5,%4\n\t{sfe|subfe} %0,%3,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "r" ((USItype)(ah)), \ "r" ((USItype)(bh)), \ "rI" ((USItype)(al)), \ @@ -787,7 +787,7 @@ do { \ do { \ USItype __m0 = (m0), __m1 = (m1); \ __asm__ ("mulhwu %0,%1,%2" \ - : "=r" ((USItype) ph) \ + : "=r" (ph) \ : "%r" (__m0), \ "r" (__m1)); \ (pl) = __m0 * __m1; \ diff --git a/lib/mpi/mpi-add.c b/lib/mpi/mpi-add.c new file mode 100644 index 000000000000..2cdae54c1bd0 --- /dev/null +++ b/lib/mpi/mpi-add.c @@ -0,0 +1,155 @@ +/* mpi-add.c - MPI functions + * Copyright (C) 1994, 1996, 1998, 2001, 2002, + * 2003 Free Software Foundation, Inc. + * + * This file is part of Libgcrypt. + * + * Note: This code is heavily based on the GNU MP Library. + * Actually it's the same code with only minor changes in the + * way the data is stored; this is to support the abstraction + * of an optional secure memory allocation which may be used + * to avoid revealing of sensitive data due to paging etc. + */ + +#include "mpi-internal.h" + +/**************** + * Add the unsigned integer V to the mpi-integer U and store the + * result in W. U and V may be the same. + */ +void mpi_add_ui(MPI w, MPI u, unsigned long v) +{ + mpi_ptr_t wp, up; + mpi_size_t usize, wsize; + int usign, wsign; + + usize = u->nlimbs; + usign = u->sign; + wsign = 0; + + /* If not space for W (and possible carry), increase space. */ + wsize = usize + 1; + if (w->alloced < wsize) + mpi_resize(w, wsize); + + /* These must be after realloc (U may be the same as W). */ + up = u->d; + wp = w->d; + + if (!usize) { /* simple */ + wp[0] = v; + wsize = v ? 1:0; + } else if (!usign) { /* mpi is not negative */ + mpi_limb_t cy; + cy = mpihelp_add_1(wp, up, usize, v); + wp[usize] = cy; + wsize = usize + cy; + } else { + /* The signs are different. Need exact comparison to determine + * which operand to subtract from which. + */ + if (usize == 1 && up[0] < v) { + wp[0] = v - up[0]; + wsize = 1; + } else { + mpihelp_sub_1(wp, up, usize, v); + /* Size can decrease with at most one limb. */ + wsize = usize - (wp[usize-1] == 0); + wsign = 1; + } + } + + w->nlimbs = wsize; + w->sign = wsign; +} + + +void mpi_add(MPI w, MPI u, MPI v) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t usize, vsize, wsize; + int usign, vsign, wsign; + + if (u->nlimbs < v->nlimbs) { /* Swap U and V. */ + usize = v->nlimbs; + usign = v->sign; + vsize = u->nlimbs; + vsign = u->sign; + wsize = usize + 1; + RESIZE_IF_NEEDED(w, wsize); + /* These must be after realloc (u or v may be the same as w). */ + up = v->d; + vp = u->d; + } else { + usize = u->nlimbs; + usign = u->sign; + vsize = v->nlimbs; + vsign = v->sign; + wsize = usize + 1; + RESIZE_IF_NEEDED(w, wsize); + /* These must be after realloc (u or v may be the same as w). */ + up = u->d; + vp = v->d; + } + wp = w->d; + wsign = 0; + + if (!vsize) { /* simple */ + MPN_COPY(wp, up, usize); + wsize = usize; + wsign = usign; + } else if (usign != vsign) { /* different sign */ + /* This test is right since USIZE >= VSIZE */ + if (usize != vsize) { + mpihelp_sub(wp, up, usize, vp, vsize); + wsize = usize; + MPN_NORMALIZE(wp, wsize); + wsign = usign; + } else if (mpihelp_cmp(up, vp, usize) < 0) { + mpihelp_sub_n(wp, vp, up, usize); + wsize = usize; + MPN_NORMALIZE(wp, wsize); + if (!usign) + wsign = 1; + } else { + mpihelp_sub_n(wp, up, vp, usize); + wsize = usize; + MPN_NORMALIZE(wp, wsize); + if (usign) + wsign = 1; + } + } else { /* U and V have same sign. Add them. */ + mpi_limb_t cy = mpihelp_add(wp, up, usize, vp, vsize); + wp[usize] = cy; + wsize = usize + cy; + if (usign) + wsign = 1; + } + + w->nlimbs = wsize; + w->sign = wsign; +} +EXPORT_SYMBOL_GPL(mpi_add); + +void mpi_sub(MPI w, MPI u, MPI v) +{ + MPI vv = mpi_copy(v); + vv->sign = !vv->sign; + mpi_add(w, u, vv); + mpi_free(vv); +} + + +void mpi_addm(MPI w, MPI u, MPI v, MPI m) +{ + mpi_add(w, u, v); + mpi_mod(w, w, m); +} +EXPORT_SYMBOL_GPL(mpi_addm); + +void mpi_subm(MPI w, MPI u, MPI v, MPI m) +{ + mpi_sub(w, u, v); + mpi_mod(w, w, m); +} +EXPORT_SYMBOL_GPL(mpi_subm); diff --git a/lib/mpi/mpi-bit.c b/lib/mpi/mpi-bit.c index 503537e08436..142b680835df 100644 --- a/lib/mpi/mpi-bit.c +++ b/lib/mpi/mpi-bit.c @@ -1,4 +1,4 @@ -/* mpi-bit.c - MPI bit level fucntions +/* mpi-bit.c - MPI bit level functions * Copyright (C) 1998, 1999 Free Software Foundation, Inc. * * This file is part of GnuPG. @@ -32,6 +32,7 @@ void mpi_normalize(MPI a) for (; a->nlimbs && !a->d[a->nlimbs - 1]; a->nlimbs--) ; } +EXPORT_SYMBOL_GPL(mpi_normalize); /**************** * Return the number of bits in A. @@ -54,3 +55,253 @@ unsigned mpi_get_nbits(MPI a) return n; } EXPORT_SYMBOL_GPL(mpi_get_nbits); + +/**************** + * Test whether bit N is set. + */ +int mpi_test_bit(MPI a, unsigned int n) +{ + unsigned int limbno, bitno; + mpi_limb_t limb; + + limbno = n / BITS_PER_MPI_LIMB; + bitno = n % BITS_PER_MPI_LIMB; + + if (limbno >= a->nlimbs) + return 0; /* too far left: this is a 0 */ + limb = a->d[limbno]; + return (limb & (A_LIMB_1 << bitno)) ? 1 : 0; +} +EXPORT_SYMBOL_GPL(mpi_test_bit); + +/**************** + * Set bit N of A. + */ +void mpi_set_bit(MPI a, unsigned int n) +{ + unsigned int i, limbno, bitno; + + limbno = n / BITS_PER_MPI_LIMB; + bitno = n % BITS_PER_MPI_LIMB; + + if (limbno >= a->nlimbs) { + for (i = a->nlimbs; i < a->alloced; i++) + a->d[i] = 0; + mpi_resize(a, limbno+1); + a->nlimbs = limbno+1; + } + a->d[limbno] |= (A_LIMB_1<<bitno); +} + +/**************** + * Set bit N of A. and clear all bits above + */ +void mpi_set_highbit(MPI a, unsigned int n) +{ + unsigned int i, limbno, bitno; + + limbno = n / BITS_PER_MPI_LIMB; + bitno = n % BITS_PER_MPI_LIMB; + + if (limbno >= a->nlimbs) { + for (i = a->nlimbs; i < a->alloced; i++) + a->d[i] = 0; + mpi_resize(a, limbno+1); + a->nlimbs = limbno+1; + } + a->d[limbno] |= (A_LIMB_1<<bitno); + for (bitno++; bitno < BITS_PER_MPI_LIMB; bitno++) + a->d[limbno] &= ~(A_LIMB_1 << bitno); + a->nlimbs = limbno+1; +} +EXPORT_SYMBOL_GPL(mpi_set_highbit); + +/**************** + * clear bit N of A and all bits above + */ +void mpi_clear_highbit(MPI a, unsigned int n) +{ + unsigned int limbno, bitno; + + limbno = n / BITS_PER_MPI_LIMB; + bitno = n % BITS_PER_MPI_LIMB; + + if (limbno >= a->nlimbs) + return; /* not allocated, therefore no need to clear bits :-) */ + + for ( ; bitno < BITS_PER_MPI_LIMB; bitno++) + a->d[limbno] &= ~(A_LIMB_1 << bitno); + a->nlimbs = limbno+1; +} + +/**************** + * Clear bit N of A. + */ +void mpi_clear_bit(MPI a, unsigned int n) +{ + unsigned int limbno, bitno; + + limbno = n / BITS_PER_MPI_LIMB; + bitno = n % BITS_PER_MPI_LIMB; + + if (limbno >= a->nlimbs) + return; /* Don't need to clear this bit, it's far too left. */ + a->d[limbno] &= ~(A_LIMB_1 << bitno); +} +EXPORT_SYMBOL_GPL(mpi_clear_bit); + + +/**************** + * Shift A by COUNT limbs to the right + * This is used only within the MPI library + */ +void mpi_rshift_limbs(MPI a, unsigned int count) +{ + mpi_ptr_t ap = a->d; + mpi_size_t n = a->nlimbs; + unsigned int i; + + if (count >= n) { + a->nlimbs = 0; + return; + } + + for (i = 0; i < n - count; i++) + ap[i] = ap[i+count]; + ap[i] = 0; + a->nlimbs -= count; +} + +/* + * Shift A by N bits to the right. + */ +void mpi_rshift(MPI x, MPI a, unsigned int n) +{ + mpi_size_t xsize; + unsigned int i; + unsigned int nlimbs = (n/BITS_PER_MPI_LIMB); + unsigned int nbits = (n%BITS_PER_MPI_LIMB); + + if (x == a) { + /* In-place operation. */ + if (nlimbs >= x->nlimbs) { + x->nlimbs = 0; + return; + } + + if (nlimbs) { + for (i = 0; i < x->nlimbs - nlimbs; i++) + x->d[i] = x->d[i+nlimbs]; + x->d[i] = 0; + x->nlimbs -= nlimbs; + } + if (x->nlimbs && nbits) + mpihelp_rshift(x->d, x->d, x->nlimbs, nbits); + } else if (nlimbs) { + /* Copy and shift by more or equal bits than in a limb. */ + xsize = a->nlimbs; + x->sign = a->sign; + RESIZE_IF_NEEDED(x, xsize); + x->nlimbs = xsize; + for (i = 0; i < a->nlimbs; i++) + x->d[i] = a->d[i]; + x->nlimbs = i; + + if (nlimbs >= x->nlimbs) { + x->nlimbs = 0; + return; + } + + if (nlimbs) { + for (i = 0; i < x->nlimbs - nlimbs; i++) + x->d[i] = x->d[i+nlimbs]; + x->d[i] = 0; + x->nlimbs -= nlimbs; + } + + if (x->nlimbs && nbits) + mpihelp_rshift(x->d, x->d, x->nlimbs, nbits); + } else { + /* Copy and shift by less than bits in a limb. */ + xsize = a->nlimbs; + x->sign = a->sign; + RESIZE_IF_NEEDED(x, xsize); + x->nlimbs = xsize; + + if (xsize) { + if (nbits) + mpihelp_rshift(x->d, a->d, x->nlimbs, nbits); + else { + /* The rshift helper function is not specified for + * NBITS==0, thus we do a plain copy here. + */ + for (i = 0; i < x->nlimbs; i++) + x->d[i] = a->d[i]; + } + } + } + MPN_NORMALIZE(x->d, x->nlimbs); +} + +/**************** + * Shift A by COUNT limbs to the left + * This is used only within the MPI library + */ +void mpi_lshift_limbs(MPI a, unsigned int count) +{ + mpi_ptr_t ap; + int n = a->nlimbs; + int i; + + if (!count || !n) + return; + + RESIZE_IF_NEEDED(a, n+count); + + ap = a->d; + for (i = n-1; i >= 0; i--) + ap[i+count] = ap[i]; + for (i = 0; i < count; i++) + ap[i] = 0; + a->nlimbs += count; +} + +/* + * Shift A by N bits to the left. + */ +void mpi_lshift(MPI x, MPI a, unsigned int n) +{ + unsigned int nlimbs = (n/BITS_PER_MPI_LIMB); + unsigned int nbits = (n%BITS_PER_MPI_LIMB); + + if (x == a && !n) + return; /* In-place shift with an amount of zero. */ + + if (x != a) { + /* Copy A to X. */ + unsigned int alimbs = a->nlimbs; + int asign = a->sign; + mpi_ptr_t xp, ap; + + RESIZE_IF_NEEDED(x, alimbs+nlimbs+1); + xp = x->d; + ap = a->d; + MPN_COPY(xp, ap, alimbs); + x->nlimbs = alimbs; + x->flags = a->flags; + x->sign = asign; + } + + if (nlimbs && !nbits) { + /* Shift a full number of limbs. */ + mpi_lshift_limbs(x, nlimbs); + } else if (n) { + /* We use a very dump approach: Shift left by the number of + * limbs plus one and than fix it up by an rshift. + */ + mpi_lshift_limbs(x, nlimbs+1); + mpi_rshift(x, x, BITS_PER_MPI_LIMB - nbits); + } + + MPN_NORMALIZE(x->d, x->nlimbs); +} diff --git a/lib/mpi/mpi-cmp.c b/lib/mpi/mpi-cmp.c index d25e9e96c310..c4cfa3ff0581 100644 --- a/lib/mpi/mpi-cmp.c +++ b/lib/mpi/mpi-cmp.c @@ -41,28 +41,54 @@ int mpi_cmp_ui(MPI u, unsigned long v) } EXPORT_SYMBOL_GPL(mpi_cmp_ui); -int mpi_cmp(MPI u, MPI v) +static int do_mpi_cmp(MPI u, MPI v, int absmode) { - mpi_size_t usize, vsize; + mpi_size_t usize; + mpi_size_t vsize; + int usign; + int vsign; int cmp; mpi_normalize(u); mpi_normalize(v); + usize = u->nlimbs; vsize = v->nlimbs; - if (!u->sign && v->sign) + usign = absmode ? 0 : u->sign; + vsign = absmode ? 0 : v->sign; + + /* Compare sign bits. */ + + if (!usign && vsign) return 1; - if (u->sign && !v->sign) + if (usign && !vsign) return -1; - if (usize != vsize && !u->sign && !v->sign) + + /* U and V are either both positive or both negative. */ + + if (usize != vsize && !usign && !vsign) return usize - vsize; - if (usize != vsize && u->sign && v->sign) - return vsize - usize; + if (usize != vsize && usign && vsign) + return vsize + usize; if (!usize) return 0; cmp = mpihelp_cmp(u->d, v->d, usize); - if (u->sign) - return -cmp; - return cmp; + if (!cmp) + return 0; + if ((cmp < 0?1:0) == (usign?1:0)) + return 1; + + return -1; +} + +int mpi_cmp(MPI u, MPI v) +{ + return do_mpi_cmp(u, v, 0); } EXPORT_SYMBOL_GPL(mpi_cmp); + +int mpi_cmpabs(MPI u, MPI v) +{ + return do_mpi_cmp(u, v, 1); +} +EXPORT_SYMBOL_GPL(mpi_cmpabs); diff --git a/lib/mpi/mpi-div.c b/lib/mpi/mpi-div.c new file mode 100644 index 000000000000..45beab8b9e9e --- /dev/null +++ b/lib/mpi/mpi-div.c @@ -0,0 +1,234 @@ +/* mpi-div.c - MPI functions + * Copyright (C) 1994, 1996, 1998, 2001, 2002, + * 2003 Free Software Foundation, Inc. + * + * This file is part of Libgcrypt. + * + * Note: This code is heavily based on the GNU MP Library. + * Actually it's the same code with only minor changes in the + * way the data is stored; this is to support the abstraction + * of an optional secure memory allocation which may be used + * to avoid revealing of sensitive data due to paging etc. + */ + +#include "mpi-internal.h" +#include "longlong.h" + +void mpi_tdiv_qr(MPI quot, MPI rem, MPI num, MPI den); +void mpi_fdiv_qr(MPI quot, MPI rem, MPI dividend, MPI divisor); + +void mpi_fdiv_r(MPI rem, MPI dividend, MPI divisor) +{ + int divisor_sign = divisor->sign; + MPI temp_divisor = NULL; + + /* We need the original value of the divisor after the remainder has been + * preliminary calculated. We have to copy it to temporary space if it's + * the same variable as REM. + */ + if (rem == divisor) { + temp_divisor = mpi_copy(divisor); + divisor = temp_divisor; + } + + mpi_tdiv_r(rem, dividend, divisor); + + if (((divisor_sign?1:0) ^ (dividend->sign?1:0)) && rem->nlimbs) + mpi_add(rem, rem, divisor); + + if (temp_divisor) + mpi_free(temp_divisor); +} + +void mpi_fdiv_q(MPI quot, MPI dividend, MPI divisor) +{ + MPI tmp = mpi_alloc(mpi_get_nlimbs(quot)); + mpi_fdiv_qr(quot, tmp, dividend, divisor); + mpi_free(tmp); +} + +void mpi_fdiv_qr(MPI quot, MPI rem, MPI dividend, MPI divisor) +{ + int divisor_sign = divisor->sign; + MPI temp_divisor = NULL; + + if (quot == divisor || rem == divisor) { + temp_divisor = mpi_copy(divisor); + divisor = temp_divisor; + } + + mpi_tdiv_qr(quot, rem, dividend, divisor); + + if ((divisor_sign ^ dividend->sign) && rem->nlimbs) { + mpi_sub_ui(quot, quot, 1); + mpi_add(rem, rem, divisor); + } + + if (temp_divisor) + mpi_free(temp_divisor); +} + +/* If den == quot, den needs temporary storage. + * If den == rem, den needs temporary storage. + * If num == quot, num needs temporary storage. + * If den has temporary storage, it can be normalized while being copied, + * i.e no extra storage should be allocated. + */ + +void mpi_tdiv_r(MPI rem, MPI num, MPI den) +{ + mpi_tdiv_qr(NULL, rem, num, den); +} + +void mpi_tdiv_qr(MPI quot, MPI rem, MPI num, MPI den) +{ + mpi_ptr_t np, dp; + mpi_ptr_t qp, rp; + mpi_size_t nsize = num->nlimbs; + mpi_size_t dsize = den->nlimbs; + mpi_size_t qsize, rsize; + mpi_size_t sign_remainder = num->sign; + mpi_size_t sign_quotient = num->sign ^ den->sign; + unsigned int normalization_steps; + mpi_limb_t q_limb; + mpi_ptr_t marker[5]; + int markidx = 0; + + /* Ensure space is enough for quotient and remainder. + * We need space for an extra limb in the remainder, because it's + * up-shifted (normalized) below. + */ + rsize = nsize + 1; + mpi_resize(rem, rsize); + + qsize = rsize - dsize; /* qsize cannot be bigger than this. */ + if (qsize <= 0) { + if (num != rem) { + rem->nlimbs = num->nlimbs; + rem->sign = num->sign; + MPN_COPY(rem->d, num->d, nsize); + } + if (quot) { + /* This needs to follow the assignment to rem, in case the + * numerator and quotient are the same. + */ + quot->nlimbs = 0; + quot->sign = 0; + } + return; + } + + if (quot) + mpi_resize(quot, qsize); + + /* Read pointers here, when reallocation is finished. */ + np = num->d; + dp = den->d; + rp = rem->d; + + /* Optimize division by a single-limb divisor. */ + if (dsize == 1) { + mpi_limb_t rlimb; + if (quot) { + qp = quot->d; + rlimb = mpihelp_divmod_1(qp, np, nsize, dp[0]); + qsize -= qp[qsize - 1] == 0; + quot->nlimbs = qsize; + quot->sign = sign_quotient; + } else + rlimb = mpihelp_mod_1(np, nsize, dp[0]); + rp[0] = rlimb; + rsize = rlimb != 0?1:0; + rem->nlimbs = rsize; + rem->sign = sign_remainder; + return; + } + + + if (quot) { + qp = quot->d; + /* Make sure QP and NP point to different objects. Otherwise the + * numerator would be gradually overwritten by the quotient limbs. + */ + if (qp == np) { /* Copy NP object to temporary space. */ + np = marker[markidx++] = mpi_alloc_limb_space(nsize); + MPN_COPY(np, qp, nsize); + } + } else /* Put quotient at top of remainder. */ + qp = rp + dsize; + + normalization_steps = count_leading_zeros(dp[dsize - 1]); + + /* Normalize the denominator, i.e. make its most significant bit set by + * shifting it NORMALIZATION_STEPS bits to the left. Also shift the + * numerator the same number of steps (to keep the quotient the same!). + */ + if (normalization_steps) { + mpi_ptr_t tp; + mpi_limb_t nlimb; + + /* Shift up the denominator setting the most significant bit of + * the most significant word. Use temporary storage not to clobber + * the original contents of the denominator. + */ + tp = marker[markidx++] = mpi_alloc_limb_space(dsize); + mpihelp_lshift(tp, dp, dsize, normalization_steps); + dp = tp; + + /* Shift up the numerator, possibly introducing a new most + * significant word. Move the shifted numerator in the remainder + * meanwhile. + */ + nlimb = mpihelp_lshift(rp, np, nsize, normalization_steps); + if (nlimb) { + rp[nsize] = nlimb; + rsize = nsize + 1; + } else + rsize = nsize; + } else { + /* The denominator is already normalized, as required. Copy it to + * temporary space if it overlaps with the quotient or remainder. + */ + if (dp == rp || (quot && (dp == qp))) { + mpi_ptr_t tp; + + tp = marker[markidx++] = mpi_alloc_limb_space(dsize); + MPN_COPY(tp, dp, dsize); + dp = tp; + } + + /* Move the numerator to the remainder. */ + if (rp != np) + MPN_COPY(rp, np, nsize); + + rsize = nsize; + } + + q_limb = mpihelp_divrem(qp, 0, rp, rsize, dp, dsize); + + if (quot) { + qsize = rsize - dsize; + if (q_limb) { + qp[qsize] = q_limb; + qsize += 1; + } + + quot->nlimbs = qsize; + quot->sign = sign_quotient; + } + + rsize = dsize; + MPN_NORMALIZE(rp, rsize); + + if (normalization_steps && rsize) { + mpihelp_rshift(rp, rp, rsize, normalization_steps); + rsize -= rp[rsize - 1] == 0?1:0; + } + + rem->nlimbs = rsize; + rem->sign = sign_remainder; + while (markidx) { + markidx--; + mpi_free_limb_space(marker[markidx]); + } +} diff --git a/lib/mpi/mpi-internal.h b/lib/mpi/mpi-internal.h index 91df5f0b70f2..554002182db1 100644 --- a/lib/mpi/mpi-internal.h +++ b/lib/mpi/mpi-internal.h @@ -52,6 +52,12 @@ typedef mpi_limb_t *mpi_ptr_t; /* pointer to a limb */ typedef int mpi_size_t; /* (must be a signed type) */ +#define RESIZE_IF_NEEDED(a, b) \ + do { \ + if ((a)->alloced < (b)) \ + mpi_resize((a), (b)); \ + } while (0) + /* Copy N limbs from S to D. */ #define MPN_COPY(d, s, n) \ do { \ @@ -60,6 +66,14 @@ typedef int mpi_size_t; /* (must be a signed type) */ (d)[_i] = (s)[_i]; \ } while (0) +#define MPN_COPY_INCR(d, s, n) \ + do { \ + mpi_size_t _i; \ + for (_i = 0; _i < (n); _i++) \ + (d)[_i] = (s)[_i]; \ + } while (0) + + #define MPN_COPY_DECR(d, s, n) \ do { \ mpi_size_t _i; \ @@ -92,6 +106,38 @@ typedef int mpi_size_t; /* (must be a signed type) */ mul_n(prodp, up, vp, size, tspace); \ } while (0); +/* Divide the two-limb number in (NH,,NL) by D, with DI being the largest + * limb not larger than (2**(2*BITS_PER_MP_LIMB))/D - (2**BITS_PER_MP_LIMB). + * If this would yield overflow, DI should be the largest possible number + * (i.e., only ones). For correct operation, the most significant bit of D + * has to be set. Put the quotient in Q and the remainder in R. + */ +#define UDIV_QRNND_PREINV(q, r, nh, nl, d, di) \ + do { \ + mpi_limb_t _ql __maybe_unused; \ + mpi_limb_t _q, _r; \ + mpi_limb_t _xh, _xl; \ + umul_ppmm(_q, _ql, (nh), (di)); \ + _q += (nh); /* DI is 2**BITS_PER_MPI_LIMB too small */ \ + umul_ppmm(_xh, _xl, _q, (d)); \ + sub_ddmmss(_xh, _r, (nh), (nl), _xh, _xl); \ + if (_xh) { \ + sub_ddmmss(_xh, _r, _xh, _r, 0, (d)); \ + _q++; \ + if (_xh) { \ + sub_ddmmss(_xh, _r, _xh, _r, 0, (d)); \ + _q++; \ + } \ + } \ + if (_r >= (d)) { \ + _r -= (d); \ + _q++; \ + } \ + (r) = _r; \ + (q) = _q; \ + } while (0) + + /*-- mpiutil.c --*/ mpi_ptr_t mpi_alloc_limb_space(unsigned nlimbs); void mpi_free_limb_space(mpi_ptr_t a); @@ -135,6 +181,8 @@ int mpihelp_mul(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t usize, void mpih_sqr_n_basecase(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t size); void mpih_sqr_n(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t size, mpi_ptr_t tspace); +void mpihelp_mul_n(mpi_ptr_t prodp, + mpi_ptr_t up, mpi_ptr_t vp, mpi_size_t size); int mpihelp_mul_karatsuba_case(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t usize, @@ -146,9 +194,14 @@ mpi_limb_t mpihelp_mul_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, mpi_size_t s1_size, mpi_limb_t s2_limb); /*-- mpih-div.c --*/ +mpi_limb_t mpihelp_mod_1(mpi_ptr_t dividend_ptr, mpi_size_t dividend_size, + mpi_limb_t divisor_limb); mpi_limb_t mpihelp_divrem(mpi_ptr_t qp, mpi_size_t qextra_limbs, mpi_ptr_t np, mpi_size_t nsize, mpi_ptr_t dp, mpi_size_t dsize); +mpi_limb_t mpihelp_divmod_1(mpi_ptr_t quot_ptr, + mpi_ptr_t dividend_ptr, mpi_size_t dividend_size, + mpi_limb_t divisor_limb); /*-- generic_mpih-[lr]shift.c --*/ mpi_limb_t mpihelp_lshift(mpi_ptr_t wp, mpi_ptr_t up, mpi_size_t usize, diff --git a/lib/mpi/mpi-inv.c b/lib/mpi/mpi-inv.c new file mode 100644 index 000000000000..61e37d18f793 --- /dev/null +++ b/lib/mpi/mpi-inv.c @@ -0,0 +1,143 @@ +/* mpi-inv.c - MPI functions + * Copyright (C) 1998, 2001, 2002, 2003 Free Software Foundation, Inc. + * + * This file is part of Libgcrypt. + * + * Libgcrypt is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * Libgcrypt is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "mpi-internal.h" + +/**************** + * Calculate the multiplicative inverse X of A mod N + * That is: Find the solution x for + * 1 = (a*x) mod n + */ +int mpi_invm(MPI x, MPI a, MPI n) +{ + /* Extended Euclid's algorithm (See TAOCP Vol II, 4.5.2, Alg X) + * modified according to Michael Penk's solution for Exercise 35 + * with further enhancement + */ + MPI u, v, u1, u2 = NULL, u3, v1, v2 = NULL, v3, t1, t2 = NULL, t3; + unsigned int k; + int sign; + int odd; + + if (!mpi_cmp_ui(a, 0)) + return 0; /* Inverse does not exists. */ + if (!mpi_cmp_ui(n, 1)) + return 0; /* Inverse does not exists. */ + + u = mpi_copy(a); + v = mpi_copy(n); + + for (k = 0; !mpi_test_bit(u, 0) && !mpi_test_bit(v, 0); k++) { + mpi_rshift(u, u, 1); + mpi_rshift(v, v, 1); + } + odd = mpi_test_bit(v, 0); + + u1 = mpi_alloc_set_ui(1); + if (!odd) + u2 = mpi_alloc_set_ui(0); + u3 = mpi_copy(u); + v1 = mpi_copy(v); + if (!odd) { + v2 = mpi_alloc(mpi_get_nlimbs(u)); + mpi_sub(v2, u1, u); /* U is used as const 1 */ + } + v3 = mpi_copy(v); + if (mpi_test_bit(u, 0)) { /* u is odd */ + t1 = mpi_alloc_set_ui(0); + if (!odd) { + t2 = mpi_alloc_set_ui(1); + t2->sign = 1; + } + t3 = mpi_copy(v); + t3->sign = !t3->sign; + goto Y4; + } else { + t1 = mpi_alloc_set_ui(1); + if (!odd) + t2 = mpi_alloc_set_ui(0); + t3 = mpi_copy(u); + } + + do { + do { + if (!odd) { + if (mpi_test_bit(t1, 0) || mpi_test_bit(t2, 0)) { + /* one is odd */ + mpi_add(t1, t1, v); + mpi_sub(t2, t2, u); + } + mpi_rshift(t1, t1, 1); + mpi_rshift(t2, t2, 1); + mpi_rshift(t3, t3, 1); + } else { + if (mpi_test_bit(t1, 0)) + mpi_add(t1, t1, v); + mpi_rshift(t1, t1, 1); + mpi_rshift(t3, t3, 1); + } +Y4: + ; + } while (!mpi_test_bit(t3, 0)); /* while t3 is even */ + + if (!t3->sign) { + mpi_set(u1, t1); + if (!odd) + mpi_set(u2, t2); + mpi_set(u3, t3); + } else { + mpi_sub(v1, v, t1); + sign = u->sign; u->sign = !u->sign; + if (!odd) + mpi_sub(v2, u, t2); + u->sign = sign; + sign = t3->sign; t3->sign = !t3->sign; + mpi_set(v3, t3); + t3->sign = sign; + } + mpi_sub(t1, u1, v1); + if (!odd) + mpi_sub(t2, u2, v2); + mpi_sub(t3, u3, v3); + if (t1->sign) { + mpi_add(t1, t1, v); + if (!odd) + mpi_sub(t2, t2, u); + } + } while (mpi_cmp_ui(t3, 0)); /* while t3 != 0 */ + /* mpi_lshift( u3, k ); */ + mpi_set(x, u1); + + mpi_free(u1); + mpi_free(v1); + mpi_free(t1); + if (!odd) { + mpi_free(u2); + mpi_free(v2); + mpi_free(t2); + } + mpi_free(u3); + mpi_free(v3); + mpi_free(t3); + + mpi_free(u); + mpi_free(v); + return 1; +} +EXPORT_SYMBOL_GPL(mpi_invm); diff --git a/lib/mpi/mpi-mod.c b/lib/mpi/mpi-mod.c new file mode 100644 index 000000000000..47bc59edd4ff --- /dev/null +++ b/lib/mpi/mpi-mod.c @@ -0,0 +1,155 @@ +/* mpi-mod.c - Modular reduction + * Copyright (C) 1998, 1999, 2001, 2002, 2003, + * 2007 Free Software Foundation, Inc. + * + * This file is part of Libgcrypt. + */ + + +#include "mpi-internal.h" +#include "longlong.h" + +/* Context used with Barrett reduction. */ +struct barrett_ctx_s { + MPI m; /* The modulus - may not be modified. */ + int m_copied; /* If true, M needs to be released. */ + int k; + MPI y; + MPI r1; /* Helper MPI. */ + MPI r2; /* Helper MPI. */ + MPI r3; /* Helper MPI allocated on demand. */ +}; + + + +void mpi_mod(MPI rem, MPI dividend, MPI divisor) +{ + mpi_fdiv_r(rem, dividend, divisor); +} + +/* This function returns a new context for Barrett based operations on + * the modulus M. This context needs to be released using + * _gcry_mpi_barrett_free. If COPY is true M will be transferred to + * the context and the user may change M. If COPY is false, M may not + * be changed until gcry_mpi_barrett_free has been called. + */ +mpi_barrett_t mpi_barrett_init(MPI m, int copy) +{ + mpi_barrett_t ctx; + MPI tmp; + + mpi_normalize(m); + ctx = kcalloc(1, sizeof(*ctx), GFP_KERNEL); + + if (copy) { + ctx->m = mpi_copy(m); + ctx->m_copied = 1; + } else + ctx->m = m; + + ctx->k = mpi_get_nlimbs(m); + tmp = mpi_alloc(ctx->k + 1); + + /* Barrett precalculation: y = floor(b^(2k) / m). */ + mpi_set_ui(tmp, 1); + mpi_lshift_limbs(tmp, 2 * ctx->k); + mpi_fdiv_q(tmp, tmp, m); + + ctx->y = tmp; + ctx->r1 = mpi_alloc(2 * ctx->k + 1); + ctx->r2 = mpi_alloc(2 * ctx->k + 1); + + return ctx; +} + +void mpi_barrett_free(mpi_barrett_t ctx) +{ + if (ctx) { + mpi_free(ctx->y); + mpi_free(ctx->r1); + mpi_free(ctx->r2); + if (ctx->r3) + mpi_free(ctx->r3); + if (ctx->m_copied) + mpi_free(ctx->m); + kfree(ctx); + } +} + + +/* R = X mod M + * + * Using Barrett reduction. Before using this function + * _gcry_mpi_barrett_init must have been called to do the + * precalculations. CTX is the context created by this precalculation + * and also conveys M. If the Barret reduction could no be done a + * straightforward reduction method is used. + * + * We assume that these conditions are met: + * Input: x =(x_2k-1 ...x_0)_b + * m =(m_k-1 ....m_0)_b with m_k-1 != 0 + * Output: r = x mod m + */ +void mpi_mod_barrett(MPI r, MPI x, mpi_barrett_t ctx) +{ + MPI m = ctx->m; + int k = ctx->k; + MPI y = ctx->y; + MPI r1 = ctx->r1; + MPI r2 = ctx->r2; + int sign; + + mpi_normalize(x); + if (mpi_get_nlimbs(x) > 2*k) { + mpi_mod(r, x, m); + return; + } + + sign = x->sign; + x->sign = 0; + + /* 1. q1 = floor( x / b^k-1) + * q2 = q1 * y + * q3 = floor( q2 / b^k+1 ) + * Actually, we don't need qx, we can work direct on r2 + */ + mpi_set(r2, x); + mpi_rshift_limbs(r2, k-1); + mpi_mul(r2, r2, y); + mpi_rshift_limbs(r2, k+1); + + /* 2. r1 = x mod b^k+1 + * r2 = q3 * m mod b^k+1 + * r = r1 - r2 + * 3. if r < 0 then r = r + b^k+1 + */ + mpi_set(r1, x); + if (r1->nlimbs > k+1) /* Quick modulo operation. */ + r1->nlimbs = k+1; + mpi_mul(r2, r2, m); + if (r2->nlimbs > k+1) /* Quick modulo operation. */ + r2->nlimbs = k+1; + mpi_sub(r, r1, r2); + + if (mpi_has_sign(r)) { + if (!ctx->r3) { + ctx->r3 = mpi_alloc(k + 2); + mpi_set_ui(ctx->r3, 1); + mpi_lshift_limbs(ctx->r3, k + 1); + } + mpi_add(r, r, ctx->r3); + } + + /* 4. while r >= m do r = r - m */ + while (mpi_cmp(r, m) >= 0) + mpi_sub(r, r, m); + + x->sign = sign; +} + + +void mpi_mul_barrett(MPI w, MPI u, MPI v, mpi_barrett_t ctx) +{ + mpi_mul(w, u, v); + mpi_mod_barrett(w, w, ctx); +} diff --git a/lib/mpi/mpi-mul.c b/lib/mpi/mpi-mul.c new file mode 100644 index 000000000000..8f5fa200f297 --- /dev/null +++ b/lib/mpi/mpi-mul.c @@ -0,0 +1,91 @@ +/* mpi-mul.c - MPI functions + * Copyright (C) 1994, 1996, 1998, 2001, 2002, + * 2003 Free Software Foundation, Inc. + * + * This file is part of Libgcrypt. + * + * Note: This code is heavily based on the GNU MP Library. + * Actually it's the same code with only minor changes in the + * way the data is stored; this is to support the abstraction + * of an optional secure memory allocation which may be used + * to avoid revealing of sensitive data due to paging etc. + */ + +#include "mpi-internal.h" + +void mpi_mul(MPI w, MPI u, MPI v) +{ + mpi_size_t usize, vsize, wsize; + mpi_ptr_t up, vp, wp; + mpi_limb_t cy; + int usign, vsign, sign_product; + int assign_wp = 0; + mpi_ptr_t tmp_limb = NULL; + + if (u->nlimbs < v->nlimbs) { + /* Swap U and V. */ + usize = v->nlimbs; + usign = v->sign; + up = v->d; + vsize = u->nlimbs; + vsign = u->sign; + vp = u->d; + } else { + usize = u->nlimbs; + usign = u->sign; + up = u->d; + vsize = v->nlimbs; + vsign = v->sign; + vp = v->d; + } + sign_product = usign ^ vsign; + wp = w->d; + + /* Ensure W has space enough to store the result. */ + wsize = usize + vsize; + if (w->alloced < wsize) { + if (wp == up || wp == vp) { + wp = mpi_alloc_limb_space(wsize); + assign_wp = 1; + } else { + mpi_resize(w, wsize); + wp = w->d; + } + } else { /* Make U and V not overlap with W. */ + if (wp == up) { + /* W and U are identical. Allocate temporary space for U. */ + up = tmp_limb = mpi_alloc_limb_space(usize); + /* Is V identical too? Keep it identical with U. */ + if (wp == vp) + vp = up; + /* Copy to the temporary space. */ + MPN_COPY(up, wp, usize); + } else if (wp == vp) { + /* W and V are identical. Allocate temporary space for V. */ + vp = tmp_limb = mpi_alloc_limb_space(vsize); + /* Copy to the temporary space. */ + MPN_COPY(vp, wp, vsize); + } + } + + if (!vsize) + wsize = 0; + else { + mpihelp_mul(wp, up, usize, vp, vsize, &cy); + wsize -= cy ? 0:1; + } + + if (assign_wp) + mpi_assign_limb_space(w, wp, wsize); + w->nlimbs = wsize; + w->sign = sign_product; + if (tmp_limb) + mpi_free_limb_space(tmp_limb); +} + +void mpi_mulm(MPI w, MPI u, MPI v, MPI m) +{ + mpi_mul(w, u, v); + mpi_tdiv_r(w, w, m); +} +EXPORT_SYMBOL_GPL(mpi_mulm); diff --git a/lib/mpi/mpi-sub-ui.c b/lib/mpi/mpi-sub-ui.c new file mode 100644 index 000000000000..b41b082b5f3e --- /dev/null +++ b/lib/mpi/mpi-sub-ui.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* mpi-sub-ui.c - Subtract an unsigned integer from an MPI. + * + * Copyright 1991, 1993, 1994, 1996, 1999-2002, 2004, 2012, 2013, 2015 + * Free Software Foundation, Inc. + * + * This file was based on the GNU MP Library source file: + * https://gmplib.org/repo/gmp-6.2/file/510b83519d1c/mpz/aors_ui.h + * + * The GNU MP Library is free software; you can redistribute it and/or modify + * it under the terms of either: + * + * * the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your + * option) any later version. + * + * or + * + * * the GNU General Public License as published by the Free Software + * Foundation; either version 2 of the License, or (at your option) any + * later version. + * + * or both in parallel, as here. + * + * The GNU MP Library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received copies of the GNU General Public License and the + * GNU Lesser General Public License along with the GNU MP Library. If not, + * see https://www.gnu.org/licenses/. + */ + +#include "mpi-internal.h" + +int mpi_sub_ui(MPI w, MPI u, unsigned long vval) +{ + if (u->nlimbs == 0) { + if (mpi_resize(w, 1) < 0) + return -ENOMEM; + w->d[0] = vval; + w->nlimbs = (vval != 0); + w->sign = (vval != 0); + return 0; + } + + /* If not space for W (and possible carry), increase space. */ + if (mpi_resize(w, u->nlimbs + 1)) + return -ENOMEM; + + if (u->sign) { + mpi_limb_t cy; + + cy = mpihelp_add_1(w->d, u->d, u->nlimbs, (mpi_limb_t) vval); + w->d[u->nlimbs] = cy; + w->nlimbs = u->nlimbs + cy; + w->sign = 1; + } else { + /* The signs are different. Need exact comparison to determine + * which operand to subtract from which. + */ + if (u->nlimbs == 1 && u->d[0] < vval) { + w->d[0] = vval - u->d[0]; + w->nlimbs = 1; + w->sign = 1; + } else { + mpihelp_sub_1(w->d, u->d, u->nlimbs, (mpi_limb_t) vval); + /* Size can decrease with at most one limb. */ + w->nlimbs = (u->nlimbs - (w->d[u->nlimbs - 1] == 0)); + w->sign = 0; + } + } + + mpi_normalize(w); + return 0; +} +EXPORT_SYMBOL_GPL(mpi_sub_ui); diff --git a/lib/mpi/mpicoder.c b/lib/mpi/mpicoder.c index eead4b339466..39c4c6731094 100644 --- a/lib/mpi/mpicoder.c +++ b/lib/mpi/mpicoder.c @@ -25,6 +25,7 @@ #include <linux/string.h> #include "mpi-internal.h" +#define MAX_EXTERN_SCAN_BYTES (16*1024*1024) #define MAX_EXTERN_MPI_BITS 16384 /** @@ -109,6 +110,112 @@ MPI mpi_read_from_buffer(const void *xbuffer, unsigned *ret_nread) } EXPORT_SYMBOL_GPL(mpi_read_from_buffer); +/**************** + * Fill the mpi VAL from the hex string in STR. + */ +int mpi_fromstr(MPI val, const char *str) +{ + int sign = 0; + int prepend_zero = 0; + int i, j, c, c1, c2; + unsigned int nbits, nbytes, nlimbs; + mpi_limb_t a; + + if (*str == '-') { + sign = 1; + str++; + } + + /* Skip optional hex prefix. */ + if (*str == '0' && str[1] == 'x') + str += 2; + + nbits = strlen(str); + if (nbits > MAX_EXTERN_SCAN_BYTES) { + mpi_clear(val); + return -EINVAL; + } + nbits *= 4; + if ((nbits % 8)) + prepend_zero = 1; + + nbytes = (nbits+7) / 8; + nlimbs = (nbytes+BYTES_PER_MPI_LIMB-1) / BYTES_PER_MPI_LIMB; + + if (val->alloced < nlimbs) + mpi_resize(val, nlimbs); + + i = BYTES_PER_MPI_LIMB - (nbytes % BYTES_PER_MPI_LIMB); + i %= BYTES_PER_MPI_LIMB; + j = val->nlimbs = nlimbs; + val->sign = sign; + for (; j > 0; j--) { + a = 0; + for (; i < BYTES_PER_MPI_LIMB; i++) { + if (prepend_zero) { + c1 = '0'; + prepend_zero = 0; + } else + c1 = *str++; + + if (!c1) { + mpi_clear(val); + return -EINVAL; + } + c2 = *str++; + if (!c2) { + mpi_clear(val); + return -EINVAL; + } + if (c1 >= '0' && c1 <= '9') + c = c1 - '0'; + else if (c1 >= 'a' && c1 <= 'f') + c = c1 - 'a' + 10; + else if (c1 >= 'A' && c1 <= 'F') + c = c1 - 'A' + 10; + else { + mpi_clear(val); + return -EINVAL; + } + c <<= 4; + if (c2 >= '0' && c2 <= '9') + c |= c2 - '0'; + else if (c2 >= 'a' && c2 <= 'f') + c |= c2 - 'a' + 10; + else if (c2 >= 'A' && c2 <= 'F') + c |= c2 - 'A' + 10; + else { + mpi_clear(val); + return -EINVAL; + } + a <<= 8; + a |= c; + } + i = 0; + val->d[j-1] = a; + } + + return 0; +} +EXPORT_SYMBOL_GPL(mpi_fromstr); + +MPI mpi_scanval(const char *string) +{ + MPI a; + + a = mpi_alloc(0); + if (!a) + return NULL; + + if (mpi_fromstr(a, string)) { + mpi_free(a); + return NULL; + } + mpi_normalize(a); + return a; +} +EXPORT_SYMBOL_GPL(mpi_scanval); + static int count_lzeros(MPI a) { mpi_limb_t alimb; @@ -127,11 +234,11 @@ static int count_lzeros(MPI a) } /** - * mpi_read_buffer() - read MPI to a bufer provided by user (msb first) + * mpi_read_buffer() - read MPI to a buffer provided by user (msb first) * * @a: a multi precision integer - * @buf: bufer to which the output will be written to. Needs to be at - * leaset mpi_get_size(a) long. + * @buf: buffer to which the output will be written to. Needs to be at + * least mpi_get_size(a) long. * @buf_len: size of the buf. * @nbytes: receives the actual length of the data written on success and * the data to-be-written on -EOVERFLOW in case buf_len was too @@ -413,3 +520,232 @@ MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int nbytes) return val; } EXPORT_SYMBOL_GPL(mpi_read_raw_from_sgl); + +/* Perform a two's complement operation on buffer P of size N bytes. */ +static void twocompl(unsigned char *p, unsigned int n) +{ + int i; + + for (i = n-1; i >= 0 && !p[i]; i--) + ; + if (i >= 0) { + if ((p[i] & 0x01)) + p[i] = (((p[i] ^ 0xfe) | 0x01) & 0xff); + else if ((p[i] & 0x02)) + p[i] = (((p[i] ^ 0xfc) | 0x02) & 0xfe); + else if ((p[i] & 0x04)) + p[i] = (((p[i] ^ 0xf8) | 0x04) & 0xfc); + else if ((p[i] & 0x08)) + p[i] = (((p[i] ^ 0xf0) | 0x08) & 0xf8); + else if ((p[i] & 0x10)) + p[i] = (((p[i] ^ 0xe0) | 0x10) & 0xf0); + else if ((p[i] & 0x20)) + p[i] = (((p[i] ^ 0xc0) | 0x20) & 0xe0); + else if ((p[i] & 0x40)) + p[i] = (((p[i] ^ 0x80) | 0x40) & 0xc0); + else + p[i] = 0x80; + + for (i--; i >= 0; i--) + p[i] ^= 0xff; + } +} + +int mpi_print(enum gcry_mpi_format format, unsigned char *buffer, + size_t buflen, size_t *nwritten, MPI a) +{ + unsigned int nbits = mpi_get_nbits(a); + size_t len; + size_t dummy_nwritten; + int negative; + + if (!nwritten) + nwritten = &dummy_nwritten; + + /* Libgcrypt does no always care to set clear the sign if the value + * is 0. For printing this is a bit of a surprise, in particular + * because if some of the formats don't support negative numbers but + * should be able to print a zero. Thus we need this extra test + * for a negative number. + */ + if (a->sign && mpi_cmp_ui(a, 0)) + negative = 1; + else + negative = 0; + + len = buflen; + *nwritten = 0; + if (format == GCRYMPI_FMT_STD) { + unsigned char *tmp; + int extra = 0; + unsigned int n; + + tmp = mpi_get_buffer(a, &n, NULL); + if (!tmp) + return -EINVAL; + + if (negative) { + twocompl(tmp, n); + if (!(*tmp & 0x80)) { + /* Need to extend the sign. */ + n++; + extra = 2; + } + } else if (n && (*tmp & 0x80)) { + /* Positive but the high bit of the returned buffer is set. + * Thus we need to print an extra leading 0x00 so that the + * output is interpreted as a positive number. + */ + n++; + extra = 1; + } + + if (buffer && n > len) { + /* The provided buffer is too short. */ + kfree(tmp); + return -E2BIG; + } + if (buffer) { + unsigned char *s = buffer; + + if (extra == 1) + *s++ = 0; + else if (extra) + *s++ = 0xff; + memcpy(s, tmp, n-!!extra); + } + kfree(tmp); + *nwritten = n; + return 0; + } else if (format == GCRYMPI_FMT_USG) { + unsigned int n = (nbits + 7)/8; + + /* Note: We ignore the sign for this format. */ + /* FIXME: for performance reasons we should put this into + * mpi_aprint because we can then use the buffer directly. + */ + + if (buffer && n > len) + return -E2BIG; + if (buffer) { + unsigned char *tmp; + + tmp = mpi_get_buffer(a, &n, NULL); + if (!tmp) + return -EINVAL; + memcpy(buffer, tmp, n); + kfree(tmp); + } + *nwritten = n; + return 0; + } else if (format == GCRYMPI_FMT_PGP) { + unsigned int n = (nbits + 7)/8; + + /* The PGP format can only handle unsigned integers. */ + if (negative) + return -EINVAL; + + if (buffer && n+2 > len) + return -E2BIG; + + if (buffer) { + unsigned char *tmp; + unsigned char *s = buffer; + + s[0] = nbits >> 8; + s[1] = nbits; + + tmp = mpi_get_buffer(a, &n, NULL); + if (!tmp) + return -EINVAL; + memcpy(s+2, tmp, n); + kfree(tmp); + } + *nwritten = n+2; + return 0; + } else if (format == GCRYMPI_FMT_SSH) { + unsigned char *tmp; + int extra = 0; + unsigned int n; + + tmp = mpi_get_buffer(a, &n, NULL); + if (!tmp) + return -EINVAL; + + if (negative) { + twocompl(tmp, n); + if (!(*tmp & 0x80)) { + /* Need to extend the sign. */ + n++; + extra = 2; + } + } else if (n && (*tmp & 0x80)) { + n++; + extra = 1; + } + + if (buffer && n+4 > len) { + kfree(tmp); + return -E2BIG; + } + + if (buffer) { + unsigned char *s = buffer; + + *s++ = n >> 24; + *s++ = n >> 16; + *s++ = n >> 8; + *s++ = n; + if (extra == 1) + *s++ = 0; + else if (extra) + *s++ = 0xff; + memcpy(s, tmp, n-!!extra); + } + kfree(tmp); + *nwritten = 4+n; + return 0; + } else if (format == GCRYMPI_FMT_HEX) { + unsigned char *tmp; + int i; + int extra = 0; + unsigned int n = 0; + + tmp = mpi_get_buffer(a, &n, NULL); + if (!tmp) + return -EINVAL; + if (!n || (*tmp & 0x80)) + extra = 2; + + if (buffer && 2*n + extra + negative + 1 > len) { + kfree(tmp); + return -E2BIG; + } + if (buffer) { + unsigned char *s = buffer; + + if (negative) + *s++ = '-'; + if (extra) { + *s++ = '0'; + *s++ = '0'; + } + + for (i = 0; i < n; i++) { + unsigned int c = tmp[i]; + + *s++ = (c >> 4) < 10 ? '0'+(c>>4) : 'A'+(c>>4)-10; + c &= 15; + *s++ = c < 10 ? '0'+c : 'A'+c-10; + } + *s++ = 0; + *nwritten = s - buffer; + } else { + *nwritten = 2*n + extra + negative + 1; + } + kfree(tmp); + return 0; + } else + return -EINVAL; +} +EXPORT_SYMBOL_GPL(mpi_print); diff --git a/lib/mpi/mpih-div.c b/lib/mpi/mpih-div.c index 913a519eb005..be70ee2e42d3 100644 --- a/lib/mpi/mpih-div.c +++ b/lib/mpi/mpih-div.c @@ -24,6 +24,150 @@ #define UDIV_TIME UMUL_TIME #endif + +mpi_limb_t +mpihelp_mod_1(mpi_ptr_t dividend_ptr, mpi_size_t dividend_size, + mpi_limb_t divisor_limb) +{ + mpi_size_t i; + mpi_limb_t n1, n0, r; + mpi_limb_t dummy __maybe_unused; + + /* Botch: Should this be handled at all? Rely on callers? */ + if (!dividend_size) + return 0; + + /* If multiplication is much faster than division, and the + * dividend is large, pre-invert the divisor, and use + * only multiplications in the inner loop. + * + * This test should be read: + * Does it ever help to use udiv_qrnnd_preinv? + * && Does what we save compensate for the inversion overhead? + */ + if (UDIV_TIME > (2 * UMUL_TIME + 6) + && (UDIV_TIME - (2 * UMUL_TIME + 6)) * dividend_size > UDIV_TIME) { + int normalization_steps; + + normalization_steps = count_leading_zeros(divisor_limb); + if (normalization_steps) { + mpi_limb_t divisor_limb_inverted; + + divisor_limb <<= normalization_steps; + + /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The + * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the + * most significant bit (with weight 2**N) implicit. + * + * Special case for DIVISOR_LIMB == 100...000. + */ + if (!(divisor_limb << 1)) + divisor_limb_inverted = ~(mpi_limb_t)0; + else + udiv_qrnnd(divisor_limb_inverted, dummy, + -divisor_limb, 0, divisor_limb); + + n1 = dividend_ptr[dividend_size - 1]; + r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps); + + /* Possible optimization: + * if (r == 0 + * && divisor_limb > ((n1 << normalization_steps) + * | (dividend_ptr[dividend_size - 2] >> ...))) + * ...one division less... + */ + for (i = dividend_size - 2; i >= 0; i--) { + n0 = dividend_ptr[i]; + UDIV_QRNND_PREINV(dummy, r, r, + ((n1 << normalization_steps) + | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))), + divisor_limb, divisor_limb_inverted); + n1 = n0; + } + UDIV_QRNND_PREINV(dummy, r, r, + n1 << normalization_steps, + divisor_limb, divisor_limb_inverted); + return r >> normalization_steps; + } else { + mpi_limb_t divisor_limb_inverted; + + /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The + * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the + * most significant bit (with weight 2**N) implicit. + * + * Special case for DIVISOR_LIMB == 100...000. + */ + if (!(divisor_limb << 1)) + divisor_limb_inverted = ~(mpi_limb_t)0; + else + udiv_qrnnd(divisor_limb_inverted, dummy, + -divisor_limb, 0, divisor_limb); + + i = dividend_size - 1; + r = dividend_ptr[i]; + + if (r >= divisor_limb) + r = 0; + else + i--; + + for ( ; i >= 0; i--) { + n0 = dividend_ptr[i]; + UDIV_QRNND_PREINV(dummy, r, r, + n0, divisor_limb, divisor_limb_inverted); + } + return r; + } + } else { + if (UDIV_NEEDS_NORMALIZATION) { + int normalization_steps; + + normalization_steps = count_leading_zeros(divisor_limb); + if (normalization_steps) { + divisor_limb <<= normalization_steps; + + n1 = dividend_ptr[dividend_size - 1]; + r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps); + + /* Possible optimization: + * if (r == 0 + * && divisor_limb > ((n1 << normalization_steps) + * | (dividend_ptr[dividend_size - 2] >> ...))) + * ...one division less... + */ + for (i = dividend_size - 2; i >= 0; i--) { + n0 = dividend_ptr[i]; + udiv_qrnnd(dummy, r, r, + ((n1 << normalization_steps) + | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))), + divisor_limb); + n1 = n0; + } + udiv_qrnnd(dummy, r, r, + n1 << normalization_steps, + divisor_limb); + return r >> normalization_steps; + } + } + /* No normalization needed, either because udiv_qrnnd doesn't require + * it, or because DIVISOR_LIMB is already normalized. + */ + i = dividend_size - 1; + r = dividend_ptr[i]; + + if (r >= divisor_limb) + r = 0; + else + i--; + + for (; i >= 0; i--) { + n0 = dividend_ptr[i]; + udiv_qrnnd(dummy, r, r, n0, divisor_limb); + } + return r; + } +} + /* Divide num (NP/NSIZE) by den (DP/DSIZE) and write * the NSIZE-DSIZE least significant quotient limbs at QP * and the DSIZE long remainder at NP. If QEXTRA_LIMBS is @@ -221,3 +365,153 @@ q_test: return most_significant_q_limb; } + +/**************** + * Divide (DIVIDEND_PTR,,DIVIDEND_SIZE) by DIVISOR_LIMB. + * Write DIVIDEND_SIZE limbs of quotient at QUOT_PTR. + * Return the single-limb remainder. + * There are no constraints on the value of the divisor. + * + * QUOT_PTR and DIVIDEND_PTR might point to the same limb. + */ + +mpi_limb_t +mpihelp_divmod_1(mpi_ptr_t quot_ptr, + mpi_ptr_t dividend_ptr, mpi_size_t dividend_size, + mpi_limb_t divisor_limb) +{ + mpi_size_t i; + mpi_limb_t n1, n0, r; + mpi_limb_t dummy __maybe_unused; + + if (!dividend_size) + return 0; + + /* If multiplication is much faster than division, and the + * dividend is large, pre-invert the divisor, and use + * only multiplications in the inner loop. + * + * This test should be read: + * Does it ever help to use udiv_qrnnd_preinv? + * && Does what we save compensate for the inversion overhead? + */ + if (UDIV_TIME > (2 * UMUL_TIME + 6) + && (UDIV_TIME - (2 * UMUL_TIME + 6)) * dividend_size > UDIV_TIME) { + int normalization_steps; + + normalization_steps = count_leading_zeros(divisor_limb); + if (normalization_steps) { + mpi_limb_t divisor_limb_inverted; + + divisor_limb <<= normalization_steps; + + /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The + * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the + * most significant bit (with weight 2**N) implicit. + */ + /* Special case for DIVISOR_LIMB == 100...000. */ + if (!(divisor_limb << 1)) + divisor_limb_inverted = ~(mpi_limb_t)0; + else + udiv_qrnnd(divisor_limb_inverted, dummy, + -divisor_limb, 0, divisor_limb); + + n1 = dividend_ptr[dividend_size - 1]; + r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps); + + /* Possible optimization: + * if (r == 0 + * && divisor_limb > ((n1 << normalization_steps) + * | (dividend_ptr[dividend_size - 2] >> ...))) + * ...one division less... + */ + for (i = dividend_size - 2; i >= 0; i--) { + n0 = dividend_ptr[i]; + UDIV_QRNND_PREINV(quot_ptr[i + 1], r, r, + ((n1 << normalization_steps) + | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))), + divisor_limb, divisor_limb_inverted); + n1 = n0; + } + UDIV_QRNND_PREINV(quot_ptr[0], r, r, + n1 << normalization_steps, + divisor_limb, divisor_limb_inverted); + return r >> normalization_steps; + } else { + mpi_limb_t divisor_limb_inverted; + + /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The + * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the + * most significant bit (with weight 2**N) implicit. + */ + /* Special case for DIVISOR_LIMB == 100...000. */ + if (!(divisor_limb << 1)) + divisor_limb_inverted = ~(mpi_limb_t) 0; + else + udiv_qrnnd(divisor_limb_inverted, dummy, + -divisor_limb, 0, divisor_limb); + + i = dividend_size - 1; + r = dividend_ptr[i]; + + if (r >= divisor_limb) + r = 0; + else + quot_ptr[i--] = 0; + + for ( ; i >= 0; i--) { + n0 = dividend_ptr[i]; + UDIV_QRNND_PREINV(quot_ptr[i], r, r, + n0, divisor_limb, divisor_limb_inverted); + } + return r; + } + } else { + if (UDIV_NEEDS_NORMALIZATION) { + int normalization_steps; + + normalization_steps = count_leading_zeros(divisor_limb); + if (normalization_steps) { + divisor_limb <<= normalization_steps; + + n1 = dividend_ptr[dividend_size - 1]; + r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps); + + /* Possible optimization: + * if (r == 0 + * && divisor_limb > ((n1 << normalization_steps) + * | (dividend_ptr[dividend_size - 2] >> ...))) + * ...one division less... + */ + for (i = dividend_size - 2; i >= 0; i--) { + n0 = dividend_ptr[i]; + udiv_qrnnd(quot_ptr[i + 1], r, r, + ((n1 << normalization_steps) + | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))), + divisor_limb); + n1 = n0; + } + udiv_qrnnd(quot_ptr[0], r, r, + n1 << normalization_steps, + divisor_limb); + return r >> normalization_steps; + } + } + /* No normalization needed, either because udiv_qrnnd doesn't require + * it, or because DIVISOR_LIMB is already normalized. + */ + i = dividend_size - 1; + r = dividend_ptr[i]; + + if (r >= divisor_limb) + r = 0; + else + quot_ptr[i--] = 0; + + for (; i >= 0; i--) { + n0 = dividend_ptr[i]; + udiv_qrnnd(quot_ptr[i], r, r, n0, divisor_limb); + } + return r; + } +} diff --git a/lib/mpi/mpih-mul.c b/lib/mpi/mpih-mul.c index a93647564054..e5f1c84e3c48 100644 --- a/lib/mpi/mpih-mul.c +++ b/lib/mpi/mpih-mul.c @@ -317,6 +317,31 @@ mpih_sqr_n(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t size, mpi_ptr_t tspace) } } + +void mpihelp_mul_n(mpi_ptr_t prodp, + mpi_ptr_t up, mpi_ptr_t vp, mpi_size_t size) +{ + if (up == vp) { + if (size < KARATSUBA_THRESHOLD) + mpih_sqr_n_basecase(prodp, up, size); + else { + mpi_ptr_t tspace; + tspace = mpi_alloc_limb_space(2 * size); + mpih_sqr_n(prodp, up, size, tspace); + mpi_free_limb_space(tspace); + } + } else { + if (size < KARATSUBA_THRESHOLD) + mul_n_basecase(prodp, up, vp, size); + else { + mpi_ptr_t tspace; + tspace = mpi_alloc_limb_space(2 * size); + mul_n(prodp, up, vp, size, tspace); + mpi_free_limb_space(tspace); + } + } +} + int mpihelp_mul_karatsuba_case(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t usize, diff --git a/lib/mpi/mpiutil.c b/lib/mpi/mpiutil.c index 20ed0f766787..bc81419f400c 100644 --- a/lib/mpi/mpiutil.c +++ b/lib/mpi/mpiutil.c @@ -20,10 +20,67 @@ #include "mpi-internal.h" +/* Constants allocated right away at startup. */ +static MPI constants[MPI_NUMBER_OF_CONSTANTS]; + +/* Initialize the MPI subsystem. This is called early and allows to + * do some initialization without taking care of threading issues. + */ +static int __init mpi_init(void) +{ + int idx; + unsigned long value; + + for (idx = 0; idx < MPI_NUMBER_OF_CONSTANTS; idx++) { + switch (idx) { + case MPI_C_ZERO: + value = 0; + break; + case MPI_C_ONE: + value = 1; + break; + case MPI_C_TWO: + value = 2; + break; + case MPI_C_THREE: + value = 3; + break; + case MPI_C_FOUR: + value = 4; + break; + case MPI_C_EIGHT: + value = 8; + break; + default: + pr_err("MPI: invalid mpi_const selector %d\n", idx); + return -EFAULT; + } + constants[idx] = mpi_alloc_set_ui(value); + constants[idx]->flags = (16|32); + } + + return 0; +} +postcore_initcall(mpi_init); + +/* Return a constant MPI descripbed by NO which is one of the + * MPI_C_xxx macros. There is no need to copy this returned value; it + * may be used directly. + */ +MPI mpi_const(enum gcry_mpi_constants no) +{ + if ((int)no < 0 || no > MPI_NUMBER_OF_CONSTANTS) + pr_err("MPI: invalid mpi_const selector %d\n", no); + if (!constants[no]) + pr_err("MPI: MPI subsystem not initialized\n"); + return constants[no]; +} +EXPORT_SYMBOL_GPL(mpi_const); + /**************** * Note: It was a bad idea to use the number of limbs to allocate * because on a alpha the limbs are large but we normally need - * integers of n bits - So we should chnage this to bits (or bytes). + * integers of n bits - So we should change this to bits (or bytes). * * But mpi_alloc is used in a lot of places :-) */ @@ -69,7 +126,7 @@ void mpi_free_limb_space(mpi_ptr_t a) if (!a) return; - kzfree(a); + kfree_sensitive(a); } void mpi_assign_limb_space(MPI a, mpi_ptr_t ap, unsigned nlimbs) @@ -91,11 +148,11 @@ int mpi_resize(MPI a, unsigned nlimbs) return 0; /* no need to do it */ if (a->d) { - p = kmalloc_array(nlimbs, sizeof(mpi_limb_t), GFP_KERNEL); + p = kcalloc(nlimbs, sizeof(mpi_limb_t), GFP_KERNEL); if (!p) return -ENOMEM; memcpy(p, a->d, a->alloced * sizeof(mpi_limb_t)); - kzfree(a->d); + kfree_sensitive(a->d); a->d = p; } else { a->d = kcalloc(nlimbs, sizeof(mpi_limb_t), GFP_KERNEL); @@ -106,13 +163,22 @@ int mpi_resize(MPI a, unsigned nlimbs) return 0; } +void mpi_clear(MPI a) +{ + if (!a) + return; + a->nlimbs = 0; + a->flags = 0; +} +EXPORT_SYMBOL_GPL(mpi_clear); + void mpi_free(MPI a) { if (!a) return; if (a->flags & 4) - kzfree(a->d); + kfree_sensitive(a->d); else mpi_free_limb_space(a->d); @@ -122,5 +188,143 @@ void mpi_free(MPI a) } EXPORT_SYMBOL_GPL(mpi_free); +/**************** + * Note: This copy function should not interpret the MPI + * but copy it transparently. + */ +MPI mpi_copy(MPI a) +{ + int i; + MPI b; + + if (a) { + b = mpi_alloc(a->nlimbs); + b->nlimbs = a->nlimbs; + b->sign = a->sign; + b->flags = a->flags; + b->flags &= ~(16|32); /* Reset the immutable and constant flags. */ + for (i = 0; i < b->nlimbs; i++) + b->d[i] = a->d[i]; + } else + b = NULL; + return b; +} + +/**************** + * This function allocates an MPI which is optimized to hold + * a value as large as the one given in the argument and allocates it + * with the same flags as A. + */ +MPI mpi_alloc_like(MPI a) +{ + MPI b; + + if (a) { + b = mpi_alloc(a->nlimbs); + b->nlimbs = 0; + b->sign = 0; + b->flags = a->flags; + } else + b = NULL; + + return b; +} + + +/* Set U into W and release U. If W is NULL only U will be released. */ +void mpi_snatch(MPI w, MPI u) +{ + if (w) { + mpi_assign_limb_space(w, u->d, u->alloced); + w->nlimbs = u->nlimbs; + w->sign = u->sign; + w->flags = u->flags; + u->alloced = 0; + u->nlimbs = 0; + u->d = NULL; + } + mpi_free(u); +} + + +MPI mpi_set(MPI w, MPI u) +{ + mpi_ptr_t wp, up; + mpi_size_t usize = u->nlimbs; + int usign = u->sign; + + if (!w) + w = mpi_alloc(mpi_get_nlimbs(u)); + RESIZE_IF_NEEDED(w, usize); + wp = w->d; + up = u->d; + MPN_COPY(wp, up, usize); + w->nlimbs = usize; + w->flags = u->flags; + w->flags &= ~(16|32); /* Reset the immutable and constant flags. */ + w->sign = usign; + return w; +} +EXPORT_SYMBOL_GPL(mpi_set); + +MPI mpi_set_ui(MPI w, unsigned long u) +{ + if (!w) + w = mpi_alloc(1); + /* FIXME: If U is 0 we have no need to resize and thus possible + * allocating the the limbs. + */ + RESIZE_IF_NEEDED(w, 1); + w->d[0] = u; + w->nlimbs = u ? 1 : 0; + w->sign = 0; + w->flags = 0; + return w; +} +EXPORT_SYMBOL_GPL(mpi_set_ui); + +MPI mpi_alloc_set_ui(unsigned long u) +{ + MPI w = mpi_alloc(1); + w->d[0] = u; + w->nlimbs = u ? 1 : 0; + w->sign = 0; + return w; +} + +/**************** + * Swap the value of A and B, when SWAP is 1. + * Leave the value when SWAP is 0. + * This implementation should be constant-time regardless of SWAP. + */ +void mpi_swap_cond(MPI a, MPI b, unsigned long swap) +{ + mpi_size_t i; + mpi_size_t nlimbs; + mpi_limb_t mask = ((mpi_limb_t)0) - swap; + mpi_limb_t x; + + if (a->alloced > b->alloced) + nlimbs = b->alloced; + else + nlimbs = a->alloced; + if (a->nlimbs > nlimbs || b->nlimbs > nlimbs) + return; + + for (i = 0; i < nlimbs; i++) { + x = mask & (a->d[i] ^ b->d[i]); + a->d[i] = a->d[i] ^ x; + b->d[i] = b->d[i] ^ x; + } + + x = mask & (a->nlimbs ^ b->nlimbs); + a->nlimbs = a->nlimbs ^ x; + b->nlimbs = b->nlimbs ^ x; + + x = mask & (a->sign ^ b->sign); + a->sign = a->sign ^ x; + b->sign = b->sign ^ x; +} + MODULE_DESCRIPTION("Multiprecision maths library"); MODULE_LICENSE("GPL"); diff --git a/lib/nlattr.c b/lib/nlattr.c index cace9b307781..86029ad5ead4 100644 --- a/lib/nlattr.c +++ b/lib/nlattr.c @@ -44,8 +44,22 @@ static const u8 nla_attr_minlen[NLA_TYPE_MAX+1] = { [NLA_S64] = sizeof(s64), }; +/* + * Nested policies might refer back to the original + * policy in some cases, and userspace could try to + * abuse that and recurse by nesting in the right + * ways. Limit recursion to avoid this problem. + */ +#define MAX_POLICY_RECURSION_DEPTH 10 + +static int __nla_validate_parse(const struct nlattr *head, int len, int maxtype, + const struct nla_policy *policy, + unsigned int validate, + struct netlink_ext_ack *extack, + struct nlattr **tb, unsigned int depth); + static int validate_nla_bitfield32(const struct nlattr *nla, - const u32 *valid_flags_mask) + const u32 valid_flags_mask) { const struct nla_bitfield32 *bf = nla_data(nla); @@ -53,11 +67,11 @@ static int validate_nla_bitfield32(const struct nlattr *nla, return -EINVAL; /*disallow invalid bit selector */ - if (bf->selector & ~*valid_flags_mask) + if (bf->selector & ~valid_flags_mask) return -EINVAL; /*disallow invalid bit values */ - if (bf->value & ~*valid_flags_mask) + if (bf->value & ~valid_flags_mask) return -EINVAL; /*disallow valid bit values that are not selected*/ @@ -70,7 +84,7 @@ static int validate_nla_bitfield32(const struct nlattr *nla, static int nla_validate_array(const struct nlattr *head, int len, int maxtype, const struct nla_policy *policy, struct netlink_ext_ack *extack, - unsigned int validate) + unsigned int validate, unsigned int depth) { const struct nlattr *entry; int rem; @@ -82,13 +96,14 @@ static int nla_validate_array(const struct nlattr *head, int len, int maxtype, continue; if (nla_len(entry) < NLA_HDRLEN) { - NL_SET_ERR_MSG_ATTR(extack, entry, - "Array element too short"); + NL_SET_ERR_MSG_ATTR_POL(extack, entry, policy, + "Array element too short"); return -ERANGE; } - ret = __nla_validate(nla_data(entry), nla_len(entry), - maxtype, policy, validate, extack); + ret = __nla_validate_parse(nla_data(entry), nla_len(entry), + maxtype, policy, validate, extack, + NULL, depth + 1); if (ret < 0) return ret; } @@ -96,17 +111,61 @@ static int nla_validate_array(const struct nlattr *head, int len, int maxtype, return 0; } -static int nla_validate_int_range(const struct nla_policy *pt, - const struct nlattr *nla, - struct netlink_ext_ack *extack) +void nla_get_range_unsigned(const struct nla_policy *pt, + struct netlink_range_validation *range) { - bool validate_min, validate_max; - s64 value; + WARN_ON_ONCE(pt->validation_type != NLA_VALIDATE_RANGE_PTR && + (pt->min < 0 || pt->max < 0)); - validate_min = pt->validation_type == NLA_VALIDATE_RANGE || - pt->validation_type == NLA_VALIDATE_MIN; - validate_max = pt->validation_type == NLA_VALIDATE_RANGE || - pt->validation_type == NLA_VALIDATE_MAX; + range->min = 0; + + switch (pt->type) { + case NLA_U8: + range->max = U8_MAX; + break; + case NLA_U16: + case NLA_BINARY: + range->max = U16_MAX; + break; + case NLA_U32: + range->max = U32_MAX; + break; + case NLA_U64: + case NLA_MSECS: + range->max = U64_MAX; + break; + default: + WARN_ON_ONCE(1); + return; + } + + switch (pt->validation_type) { + case NLA_VALIDATE_RANGE: + case NLA_VALIDATE_RANGE_WARN_TOO_LONG: + range->min = pt->min; + range->max = pt->max; + break; + case NLA_VALIDATE_RANGE_PTR: + *range = *pt->range; + break; + case NLA_VALIDATE_MIN: + range->min = pt->min; + break; + case NLA_VALIDATE_MAX: + range->max = pt->max; + break; + default: + break; + } +} + +static int nla_validate_range_unsigned(const struct nla_policy *pt, + const struct nlattr *nla, + struct netlink_ext_ack *extack, + unsigned int validate) +{ + struct netlink_range_validation range; + u64 value; switch (pt->type) { case NLA_U8: @@ -118,6 +177,101 @@ static int nla_validate_int_range(const struct nla_policy *pt, case NLA_U32: value = nla_get_u32(nla); break; + case NLA_U64: + case NLA_MSECS: + value = nla_get_u64(nla); + break; + case NLA_BINARY: + value = nla_len(nla); + break; + default: + return -EINVAL; + } + + nla_get_range_unsigned(pt, &range); + + if (pt->validation_type == NLA_VALIDATE_RANGE_WARN_TOO_LONG && + pt->type == NLA_BINARY && value > range.max) { + pr_warn_ratelimited("netlink: '%s': attribute type %d has an invalid length.\n", + current->comm, pt->type); + if (validate & NL_VALIDATE_STRICT_ATTRS) { + NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt, + "invalid attribute length"); + return -EINVAL; + } + + /* this assumes min <= max (don't validate against min) */ + return 0; + } + + if (value < range.min || value > range.max) { + bool binary = pt->type == NLA_BINARY; + + if (binary) + NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt, + "binary attribute size out of range"); + else + NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt, + "integer out of range"); + + return -ERANGE; + } + + return 0; +} + +void nla_get_range_signed(const struct nla_policy *pt, + struct netlink_range_validation_signed *range) +{ + switch (pt->type) { + case NLA_S8: + range->min = S8_MIN; + range->max = S8_MAX; + break; + case NLA_S16: + range->min = S16_MIN; + range->max = S16_MAX; + break; + case NLA_S32: + range->min = S32_MIN; + range->max = S32_MAX; + break; + case NLA_S64: + range->min = S64_MIN; + range->max = S64_MAX; + break; + default: + WARN_ON_ONCE(1); + return; + } + + switch (pt->validation_type) { + case NLA_VALIDATE_RANGE: + range->min = pt->min; + range->max = pt->max; + break; + case NLA_VALIDATE_RANGE_PTR: + *range = *pt->range_signed; + break; + case NLA_VALIDATE_MIN: + range->min = pt->min; + break; + case NLA_VALIDATE_MAX: + range->max = pt->max; + break; + default: + break; + } +} + +static int nla_validate_int_range_signed(const struct nla_policy *pt, + const struct nlattr *nla, + struct netlink_ext_ack *extack) +{ + struct netlink_range_validation_signed range; + s64 value; + + switch (pt->type) { case NLA_S8: value = nla_get_s8(nla); break; @@ -130,25 +284,71 @@ static int nla_validate_int_range(const struct nla_policy *pt, case NLA_S64: value = nla_get_s64(nla); break; + default: + return -EINVAL; + } + + nla_get_range_signed(pt, &range); + + if (value < range.min || value > range.max) { + NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt, + "integer out of range"); + return -ERANGE; + } + + return 0; +} + +static int nla_validate_int_range(const struct nla_policy *pt, + const struct nlattr *nla, + struct netlink_ext_ack *extack, + unsigned int validate) +{ + switch (pt->type) { + case NLA_U8: + case NLA_U16: + case NLA_U32: case NLA_U64: - /* treat this one specially, since it may not fit into s64 */ - if ((validate_min && nla_get_u64(nla) < pt->min) || - (validate_max && nla_get_u64(nla) > pt->max)) { - NL_SET_ERR_MSG_ATTR(extack, nla, - "integer out of range"); - return -ERANGE; - } - return 0; + case NLA_MSECS: + case NLA_BINARY: + return nla_validate_range_unsigned(pt, nla, extack, validate); + case NLA_S8: + case NLA_S16: + case NLA_S32: + case NLA_S64: + return nla_validate_int_range_signed(pt, nla, extack); default: WARN_ON(1); return -EINVAL; } +} - if ((validate_min && value < pt->min) || - (validate_max && value > pt->max)) { - NL_SET_ERR_MSG_ATTR(extack, nla, - "integer out of range"); - return -ERANGE; +static int nla_validate_mask(const struct nla_policy *pt, + const struct nlattr *nla, + struct netlink_ext_ack *extack) +{ + u64 value; + + switch (pt->type) { + case NLA_U8: + value = nla_get_u8(nla); + break; + case NLA_U16: + value = nla_get_u16(nla); + break; + case NLA_U32: + value = nla_get_u32(nla); + break; + case NLA_U64: + value = nla_get_u64(nla); + break; + default: + return -EINVAL; + } + + if (value & ~(u64)pt->mask) { + NL_SET_ERR_MSG_ATTR(extack, nla, "reserved bit set"); + return -EINVAL; } return 0; @@ -156,7 +356,7 @@ static int nla_validate_int_range(const struct nla_policy *pt, static int validate_nla(const struct nlattr *nla, int maxtype, const struct nla_policy *policy, unsigned int validate, - struct netlink_ext_ack *extack) + struct netlink_ext_ack *extack, unsigned int depth) { u16 strict_start_type = policy[0].strict_start_type; const struct nla_policy *pt; @@ -173,13 +373,12 @@ static int validate_nla(const struct nlattr *nla, int maxtype, BUG_ON(pt->type > NLA_TYPE_MAX); - if ((nla_attr_len[pt->type] && attrlen != nla_attr_len[pt->type]) || - (pt->type == NLA_EXACT_LEN_WARN && attrlen != pt->len)) { + if (nla_attr_len[pt->type] && attrlen != nla_attr_len[pt->type]) { pr_warn_ratelimited("netlink: '%s': attribute type %d has an invalid length.\n", current->comm, type); if (validate & NL_VALIDATE_STRICT_ATTRS) { - NL_SET_ERR_MSG_ATTR(extack, nla, - "invalid attribute length"); + NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt, + "invalid attribute length"); return -EINVAL; } } @@ -187,28 +386,23 @@ static int validate_nla(const struct nlattr *nla, int maxtype, if (validate & NL_VALIDATE_NESTED) { if ((pt->type == NLA_NESTED || pt->type == NLA_NESTED_ARRAY) && !(nla->nla_type & NLA_F_NESTED)) { - NL_SET_ERR_MSG_ATTR(extack, nla, - "NLA_F_NESTED is missing"); + NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt, + "NLA_F_NESTED is missing"); return -EINVAL; } if (pt->type != NLA_NESTED && pt->type != NLA_NESTED_ARRAY && pt->type != NLA_UNSPEC && (nla->nla_type & NLA_F_NESTED)) { - NL_SET_ERR_MSG_ATTR(extack, nla, - "NLA_F_NESTED not expected"); + NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt, + "NLA_F_NESTED not expected"); return -EINVAL; } } switch (pt->type) { - case NLA_EXACT_LEN: - if (attrlen != pt->len) - goto out_err; - break; - case NLA_REJECT: - if (extack && pt->validation_data) { + if (extack && pt->reject_message) { NL_SET_BAD_ATTR(extack, nla); - extack->_msg = pt->validation_data; + extack->_msg = pt->reject_message; return -EINVAL; } err = -EINVAL; @@ -223,7 +417,7 @@ static int validate_nla(const struct nlattr *nla, int maxtype, if (attrlen != sizeof(struct nla_bitfield32)) goto out_err; - err = validate_nla_bitfield32(nla, pt->validation_data); + err = validate_nla_bitfield32(nla, pt->bitfield32_valid); if (err) goto out_err; break; @@ -238,7 +432,7 @@ static int validate_nla(const struct nlattr *nla, int maxtype, err = -EINVAL; goto out_err; } - /* fall through */ + fallthrough; case NLA_STRING: if (attrlen < 1) @@ -268,10 +462,11 @@ static int validate_nla(const struct nlattr *nla, int maxtype, break; if (attrlen < NLA_HDRLEN) goto out_err; - if (pt->validation_data) { - err = __nla_validate(nla_data(nla), nla_len(nla), pt->len, - pt->validation_data, validate, - extack); + if (pt->nested_policy) { + err = __nla_validate_parse(nla_data(nla), nla_len(nla), + pt->len, pt->nested_policy, + validate, extack, NULL, + depth + 1); if (err < 0) { /* * return directly to preserve the inner @@ -289,12 +484,12 @@ static int validate_nla(const struct nlattr *nla, int maxtype, break; if (attrlen < NLA_HDRLEN) goto out_err; - if (pt->validation_data) { + if (pt->nested_policy) { int err; err = nla_validate_array(nla_data(nla), nla_len(nla), - pt->len, pt->validation_data, - extack, validate); + pt->len, pt->nested_policy, + extack, validate, depth); if (err < 0) { /* * return directly to preserve the inner @@ -311,8 +506,6 @@ static int validate_nla(const struct nlattr *nla, int maxtype, "Unsupported attribute"); return -EINVAL; } - /* fall through */ - case NLA_MIN_LEN: if (attrlen < pt->len) goto out_err; break; @@ -332,10 +525,17 @@ static int validate_nla(const struct nlattr *nla, int maxtype, case NLA_VALIDATE_NONE: /* nothing to do */ break; + case NLA_VALIDATE_RANGE_PTR: case NLA_VALIDATE_RANGE: + case NLA_VALIDATE_RANGE_WARN_TOO_LONG: case NLA_VALIDATE_MIN: case NLA_VALIDATE_MAX: - err = nla_validate_int_range(pt, nla, extack); + err = nla_validate_int_range(pt, nla, extack, validate); + if (err) + return err; + break; + case NLA_VALIDATE_MASK: + err = nla_validate_mask(pt, nla, extack); if (err) return err; break; @@ -350,7 +550,8 @@ static int validate_nla(const struct nlattr *nla, int maxtype, return 0; out_err: - NL_SET_ERR_MSG_ATTR(extack, nla, "Attribute failed policy validation"); + NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt, + "Attribute failed policy validation"); return err; } @@ -358,11 +559,17 @@ static int __nla_validate_parse(const struct nlattr *head, int len, int maxtype, const struct nla_policy *policy, unsigned int validate, struct netlink_ext_ack *extack, - struct nlattr **tb) + struct nlattr **tb, unsigned int depth) { const struct nlattr *nla; int rem; + if (depth >= MAX_POLICY_RECURSION_DEPTH) { + NL_SET_ERR_MSG(extack, + "allowed policy recursion depth exceeded"); + return -EINVAL; + } + if (tb) memset(tb, 0, sizeof(struct nlattr *) * (maxtype + 1)); @@ -379,7 +586,7 @@ static int __nla_validate_parse(const struct nlattr *head, int len, int maxtype, } if (policy) { int err = validate_nla(nla, maxtype, policy, - validate, extack); + validate, extack, depth); if (err < 0) return err; @@ -412,7 +619,7 @@ static int __nla_validate_parse(const struct nlattr *head, int len, int maxtype, * Validates all attributes in the specified attribute stream against the * specified policy. Validation depends on the validate flags passed, see * &enum netlink_validation for more details on that. - * See documenation of struct nla_policy for more details. + * See documentation of struct nla_policy for more details. * * Returns 0 on success or a negative error code. */ @@ -421,12 +628,12 @@ int __nla_validate(const struct nlattr *head, int len, int maxtype, struct netlink_ext_ack *extack) { return __nla_validate_parse(head, len, maxtype, policy, validate, - extack, NULL); + extack, NULL, 0); } EXPORT_SYMBOL(__nla_validate); /** - * nla_policy_len - Determin the max. length of a policy + * nla_policy_len - Determine the max. length of a policy * @policy: policy to use * @n: number of policies * @@ -476,7 +683,7 @@ int __nla_parse(struct nlattr **tb, int maxtype, struct netlink_ext_ack *extack) { return __nla_validate_parse(head, len, maxtype, policy, validate, - extack, tb); + extack, tb, 0); } EXPORT_SYMBOL(__nla_parse); @@ -502,35 +709,47 @@ struct nlattr *nla_find(const struct nlattr *head, int len, int attrtype) EXPORT_SYMBOL(nla_find); /** - * nla_strlcpy - Copy string attribute payload into a sized buffer - * @dst: where to copy the string to - * @nla: attribute to copy the string from - * @dstsize: size of destination buffer + * nla_strscpy - Copy string attribute payload into a sized buffer + * @dst: Where to copy the string to. + * @nla: Attribute to copy the string from. + * @dstsize: Size of destination buffer. * * Copies at most dstsize - 1 bytes into the destination buffer. - * The result is always a valid NUL-terminated string. Unlike - * strlcpy the destination buffer is always padded out. + * Unlike strlcpy the destination buffer is always padded out. * - * Returns the length of the source buffer. + * Return: + * * srclen - Returns @nla length (not including the trailing %NUL). + * * -E2BIG - If @dstsize is 0 or greater than U16_MAX or @nla length greater + * than @dstsize. */ -size_t nla_strlcpy(char *dst, const struct nlattr *nla, size_t dstsize) +ssize_t nla_strscpy(char *dst, const struct nlattr *nla, size_t dstsize) { size_t srclen = nla_len(nla); char *src = nla_data(nla); + ssize_t ret; + size_t len; + + if (dstsize == 0 || WARN_ON_ONCE(dstsize > U16_MAX)) + return -E2BIG; if (srclen > 0 && src[srclen - 1] == '\0') srclen--; - if (dstsize > 0) { - size_t len = (srclen >= dstsize) ? dstsize - 1 : srclen; - - memset(dst, 0, dstsize); - memcpy(dst, src, len); + if (srclen >= dstsize) { + len = dstsize - 1; + ret = -E2BIG; + } else { + len = srclen; + ret = len; } - return srclen; + memcpy(dst, src, len); + /* Zero pad end of dst. */ + memset(dst + len, 0, dstsize - len); + + return ret; } -EXPORT_SYMBOL(nla_strlcpy); +EXPORT_SYMBOL(nla_strscpy); /** * nla_strdup - Copy string attribute payload into a newly allocated buffer @@ -609,7 +828,7 @@ int nla_strcmp(const struct nlattr *nla, const char *str) int attrlen = nla_len(nla); int d; - if (attrlen > 0 && buf[attrlen - 1] == '\0') + while (attrlen > 0 && buf[attrlen - 1] == '\0') attrlen--; d = attrlen - len; @@ -664,8 +883,7 @@ EXPORT_SYMBOL(__nla_reserve); struct nlattr *__nla_reserve_64bit(struct sk_buff *skb, int attrtype, int attrlen, int padattr) { - if (nla_need_padding_for_64bit(skb)) - nla_align_64bit(skb, padattr); + nla_align_64bit(skb, padattr); return __nla_reserve(skb, attrtype, attrlen); } diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c index 15ca78e1c7d4..f9e89001b52e 100644 --- a/lib/nmi_backtrace.c +++ b/lib/nmi_backtrace.c @@ -75,22 +75,26 @@ void nmi_trigger_cpumask_backtrace(const cpumask_t *mask, touch_softlockup_watchdog(); } - /* - * Force flush any remote buffers that might be stuck in IRQ context - * and therefore could not run their irq_work. - */ - printk_safe_flush(); - clear_bit_unlock(0, &backtrace_flag); put_cpu(); } +// Dump stacks even for idle CPUs. +static bool backtrace_idle; +module_param(backtrace_idle, bool, 0644); + bool nmi_cpu_backtrace(struct pt_regs *regs) { int cpu = smp_processor_id(); + unsigned long flags; if (cpumask_test_cpu(cpu, to_cpumask(backtrace_mask))) { - if (regs && cpu_in_idle(instruction_pointer(regs))) { + /* + * Allow nested NMI backtraces while serializing + * against other CPUs. + */ + printk_cpu_lock_irqsave(flags); + if (!READ_ONCE(backtrace_idle) && regs && cpu_in_idle(instruction_pointer(regs))) { pr_warn("NMI backtrace for cpu %d skipped: idling at %pS\n", cpu, (void *)instruction_pointer(regs)); } else { @@ -100,6 +104,7 @@ bool nmi_cpu_backtrace(struct pt_regs *regs) else dump_stack(); } + printk_cpu_unlock_irqrestore(flags); cpumask_clear_cpu(cpu, to_cpumask(backtrace_mask)); return true; } diff --git a/lib/objagg.c b/lib/objagg.c index 55621fb82e0a..5e1676ccdadd 100644 --- a/lib/objagg.c +++ b/lib/objagg.c @@ -28,7 +28,7 @@ struct objagg_hints_node { struct objagg_hints_node *parent; unsigned int root_id; struct objagg_obj_stats_info stats_info; - unsigned long obj[0]; + unsigned long obj[]; }; static struct objagg_hints_node * @@ -66,7 +66,7 @@ struct objagg_obj { * including nested objects */ struct objagg_obj_stats stats; - unsigned long obj[0]; + unsigned long obj[]; }; static unsigned int objagg_obj_ref_inc(struct objagg_obj *objagg_obj) diff --git a/lib/oid_registry.c b/lib/oid_registry.c index f7ad43f28579..e592d48b1974 100644 --- a/lib/oid_registry.c +++ b/lib/oid_registry.c @@ -11,6 +11,7 @@ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/bug.h> +#include <linux/asn1.h> #include "oid_registry_data.c" MODULE_DESCRIPTION("OID Registry"); @@ -92,6 +93,29 @@ enum OID look_up_OID(const void *data, size_t datasize) } EXPORT_SYMBOL_GPL(look_up_OID); +/** + * parse_OID - Parse an OID from a bytestream + * @data: Binary representation of the header + OID + * @datasize: Size of the binary representation + * @oid: Pointer to oid to return result + * + * Parse an OID from a bytestream that holds the OID in the format + * ASN1_OID | length | oid. The length indicator must equal to datasize - 2. + * -EBADMSG is returned if the bytestream is too short. + */ +int parse_OID(const void *data, size_t datasize, enum OID *oid) +{ + const unsigned char *v = data; + + /* we need 2 bytes of header and at least 1 byte for oid */ + if (datasize < 3 || v[0] != ASN1_OID || v[1] != datasize - 2) + return -EBADMSG; + + *oid = look_up_OID(data + 2, datasize - 2); + return 0; +} +EXPORT_SYMBOL_GPL(parse_OID); + /* * sprint_OID - Print an Object Identifier into a buffer * @data: The encoded OID to print @@ -100,7 +124,7 @@ EXPORT_SYMBOL_GPL(look_up_OID); * @bufsize: The size of the buffer * * The OID is rendered into the buffer in "a.b.c.d" format and the number of - * bytes is returned. -EBADMSG is returned if the data could not be intepreted + * bytes is returned. -EBADMSG is returned if the data could not be interpreted * and -ENOBUFS if the buffer was too small. */ int sprint_oid(const void *data, size_t datasize, char *buffer, size_t bufsize) diff --git a/lib/once.c b/lib/once.c index 8b7d6235217e..59149bf3bfb4 100644 --- a/lib/once.c +++ b/lib/once.c @@ -3,10 +3,12 @@ #include <linux/spinlock.h> #include <linux/once.h> #include <linux/random.h> +#include <linux/module.h> struct once_work { struct work_struct work; struct static_key_true *key; + struct module *module; }; static void once_deferred(struct work_struct *w) @@ -16,10 +18,11 @@ static void once_deferred(struct work_struct *w) work = container_of(w, struct once_work, work); BUG_ON(!static_key_enabled(work->key)); static_branch_disable(work->key); + module_put(work->module); kfree(work); } -static void once_disable_jump(struct static_key_true *key) +static void once_disable_jump(struct static_key_true *key, struct module *mod) { struct once_work *w; @@ -29,6 +32,8 @@ static void once_disable_jump(struct static_key_true *key) INIT_WORK(&w->work, once_deferred); w->key = key; + w->module = mod; + __module_get(mod); schedule_work(&w->work); } @@ -53,11 +58,11 @@ bool __do_once_start(bool *done, unsigned long *flags) EXPORT_SYMBOL(__do_once_start); void __do_once_done(bool *done, struct static_key_true *once_key, - unsigned long *flags) + unsigned long *flags, struct module *mod) __releases(once_lock) { *done = true; spin_unlock_irqrestore(&once_lock, *flags); - once_disable_jump(once_key); + once_disable_jump(once_key, mod); } EXPORT_SYMBOL(__do_once_done); diff --git a/lib/packing.c b/lib/packing.c index 50d1e9f2f5a7..9a72f4bbf0e2 100644 --- a/lib/packing.c +++ b/lib/packing.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 -/* Copyright (c) 2016-2018, NXP Semiconductors +/* Copyright 2016-2018 NXP * Copyright (c) 2018-2019, Vladimir Oltean <olteanv@gmail.com> */ #include <linux/packing.h> @@ -73,6 +73,7 @@ static void adjust_for_msb_right_quirk(u64 *to_write, int *box_start_bit, * @endbit: The index (in logical notation, compensated for quirks) where * the packed value ends within pbuf. Must be smaller than, or equal * to, startbit. + * @pbuflen: The length in bytes of the packed buffer pointed to by @pbuf. * @op: If PACK, then uval will be treated as const pointer and copied (packed) * into pbuf, between startbit and endbit. * If UNPACK, then pbuf will be treated as const pointer and the logical diff --git a/lib/parman.c b/lib/parman.c index c6e42a8db824..3f8f8d422e62 100644 --- a/lib/parman.c +++ b/lib/parman.c @@ -85,7 +85,6 @@ static int parman_shrink(struct parman *parman) } static bool parman_prio_used(struct parman_prio *prio) - { return !list_empty(&prio->item_list); } @@ -298,7 +297,7 @@ EXPORT_SYMBOL(parman_destroy); * parman_prio_init - initializes a parman priority chunk * @parman: parman instance * @prio: parman prio structure to be initialized - * @prority: desired priority of the chunk + * @priority: desired priority of the chunk * * Note: all locking must be provided by the caller. * @@ -357,7 +356,7 @@ int parman_item_add(struct parman *parman, struct parman_prio *prio, EXPORT_SYMBOL(parman_item_add); /** - * parman_item_del - deletes parman item + * parman_item_remove - deletes parman item * @parman: parman instance * @prio: parman prio instance to delete the item from * @item: parman item instance diff --git a/lib/parser.c b/lib/parser.c index f5b3e5d7a7f9..bcb23484100e 100644 --- a/lib/parser.c +++ b/lib/parser.c @@ -6,12 +6,13 @@ #include <linux/ctype.h> #include <linux/types.h> #include <linux/export.h> +#include <linux/kstrtox.h> #include <linux/parser.h> #include <linux/slab.h> #include <linux/string.h> /** - * match_one: - Determines if a string matches a simple pattern + * match_one - Determines if a string matches a simple pattern * @s: the string to examine for presence of the pattern * @p: the string containing the pattern * @args: array of %MAX_OPT_ARGS &substring_t elements. Used to return match @@ -89,7 +90,7 @@ static int match_one(char *s, const char *p, substring_t args[]) } /** - * match_token: - Find a token (and optional args) in a string + * match_token - Find a token (and optional args) in a string * @s: the string to examine for token/argument pairs * @table: match_table_t describing the set of allowed option tokens and the * arguments that may be associated with them. Must be terminated with a @@ -98,7 +99,7 @@ static int match_one(char *s, const char *p, substring_t args[]) * locations. * * Description: Detects which if any of a set of token strings has been passed - * to it. Tokens can include up to MAX_OPT_ARGS instances of basic c-style + * to it. Tokens can include up to %MAX_OPT_ARGS instances of basic c-style * format identifiers which will be taken into account when matching the * tokens, and whose locations will be returned in the @args array. */ @@ -114,14 +115,16 @@ int match_token(char *s, const match_table_t table, substring_t args[]) EXPORT_SYMBOL(match_token); /** - * match_number: scan a number in the given base from a substring_t + * match_number - scan a number in the given base from a substring_t * @s: substring to be scanned * @result: resulting integer on success * @base: base to use when converting string * * Description: Given a &substring_t and a base, attempts to parse the substring - * as a number in that base. On success, sets @result to the integer represented - * by the string and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. + * as a number in that base. + * + * Return: On success, sets @result to the integer represented by the + * string and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. */ static int match_number(substring_t *s, int *result, int base) { @@ -147,14 +150,16 @@ static int match_number(substring_t *s, int *result, int base) } /** - * match_u64int: scan a number in the given base from a substring_t + * match_u64int - scan a number in the given base from a substring_t * @s: substring to be scanned * @result: resulting u64 on success * @base: base to use when converting string * * Description: Given a &substring_t and a base, attempts to parse the substring - * as a number in that base. On success, sets @result to the integer represented - * by the string and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. + * as a number in that base. + * + * Return: On success, sets @result to the integer represented by the + * string and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. */ static int match_u64int(substring_t *s, u64 *result, int base) { @@ -174,13 +179,14 @@ static int match_u64int(substring_t *s, u64 *result, int base) } /** - * match_int: - scan a decimal representation of an integer from a substring_t + * match_int - scan a decimal representation of an integer from a substring_t * @s: substring_t to be scanned * @result: resulting integer on success * - * Description: Attempts to parse the &substring_t @s as a decimal integer. On - * success, sets @result to the integer represented by the string and returns 0. - * Returns -ENOMEM, -EINVAL, or -ERANGE on failure. + * Description: Attempts to parse the &substring_t @s as a decimal integer. + * + * Return: On success, sets @result to the integer represented by the string + * and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. */ int match_int(substring_t *s, int *result) { @@ -189,15 +195,39 @@ int match_int(substring_t *s, int *result) EXPORT_SYMBOL(match_int); /** - * match_u64: - scan a decimal representation of a u64 from + * match_uint - scan a decimal representation of an integer from a substring_t + * @s: substring_t to be scanned + * @result: resulting integer on success + * + * Description: Attempts to parse the &substring_t @s as a decimal integer. + * + * Return: On success, sets @result to the integer represented by the string + * and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. + */ +int match_uint(substring_t *s, unsigned int *result) +{ + int err = -ENOMEM; + char *buf = match_strdup(s); + + if (buf) { + err = kstrtouint(buf, 10, result); + kfree(buf); + } + return err; +} +EXPORT_SYMBOL(match_uint); + +/** + * match_u64 - scan a decimal representation of a u64 from * a substring_t * @s: substring_t to be scanned * @result: resulting unsigned long long on success * * Description: Attempts to parse the &substring_t @s as a long decimal - * integer. On success, sets @result to the integer represented by the - * string and returns 0. - * Returns -ENOMEM, -EINVAL, or -ERANGE on failure. + * integer. + * + * Return: On success, sets @result to the integer represented by the string + * and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. */ int match_u64(substring_t *s, u64 *result) { @@ -206,13 +236,14 @@ int match_u64(substring_t *s, u64 *result) EXPORT_SYMBOL(match_u64); /** - * match_octal: - scan an octal representation of an integer from a substring_t + * match_octal - scan an octal representation of an integer from a substring_t * @s: substring_t to be scanned * @result: resulting integer on success * - * Description: Attempts to parse the &substring_t @s as an octal integer. On - * success, sets @result to the integer represented by the string and returns - * 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. + * Description: Attempts to parse the &substring_t @s as an octal integer. + * + * Return: On success, sets @result to the integer represented by the string + * and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. */ int match_octal(substring_t *s, int *result) { @@ -221,13 +252,14 @@ int match_octal(substring_t *s, int *result) EXPORT_SYMBOL(match_octal); /** - * match_hex: - scan a hex representation of an integer from a substring_t + * match_hex - scan a hex representation of an integer from a substring_t * @s: substring_t to be scanned * @result: resulting integer on success * * Description: Attempts to parse the &substring_t @s as a hexadecimal integer. - * On success, sets @result to the integer represented by the string and - * returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. + * + * Return: On success, sets @result to the integer represented by the string + * and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure. */ int match_hex(substring_t *s, int *result) { @@ -236,15 +268,16 @@ int match_hex(substring_t *s, int *result) EXPORT_SYMBOL(match_hex); /** - * match_wildcard: - parse if a string matches given wildcard pattern + * match_wildcard - parse if a string matches given wildcard pattern * @pattern: wildcard pattern * @str: the string to be parsed * * Description: Parse the string @str to check if matches wildcard - * pattern @pattern. The pattern may contain two type wildcardes: + * pattern @pattern. The pattern may contain two types of wildcards: * '*' - matches zero or more characters * '?' - matches one character - * If it's matched, return true, else return false. + * + * Return: If the @str matches the @pattern, return true, else return false. */ bool match_wildcard(const char *pattern, const char *str) { @@ -287,14 +320,16 @@ bool match_wildcard(const char *pattern, const char *str) EXPORT_SYMBOL(match_wildcard); /** - * match_strlcpy: - Copy the characters from a substring_t to a sized buffer + * match_strlcpy - Copy the characters from a substring_t to a sized buffer * @dest: where to copy to * @src: &substring_t to copy * @size: size of destination buffer * * Description: Copy the characters in &substring_t @src to the * c-style string @dest. Copy no more than @size - 1 characters, plus - * the terminating NUL. Return length of @src. + * the terminating NUL. + * + * Return: length of @src. */ size_t match_strlcpy(char *dest, const substring_t *src, size_t size) { @@ -310,12 +345,15 @@ size_t match_strlcpy(char *dest, const substring_t *src, size_t size) EXPORT_SYMBOL(match_strlcpy); /** - * match_strdup: - allocate a new string with the contents of a substring_t + * match_strdup - allocate a new string with the contents of a substring_t * @s: &substring_t to copy * * Description: Allocates and returns a string filled with the contents of * the &substring_t @s. The caller is responsible for freeing the returned * string with kfree(). + * + * Return: the address of the newly allocated NUL-terminated string or + * %NULL on error. */ char *match_strdup(const substring_t *s) { diff --git a/lib/pci_iomap.c b/lib/pci_iomap.c index 2d3eb1cb73b8..ce39ce9f3526 100644 --- a/lib/pci_iomap.c +++ b/lib/pci_iomap.c @@ -134,4 +134,47 @@ void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen) return pci_iomap_wc_range(dev, bar, 0, maxlen); } EXPORT_SYMBOL_GPL(pci_iomap_wc); + +/* + * pci_iounmap() somewhat illogically comes from lib/iomap.c for the + * CONFIG_GENERIC_IOMAP case, because that's the code that knows about + * the different IOMAP ranges. + * + * But if the architecture does not use the generic iomap code, and if + * it has _not_ defined it's own private pci_iounmap function, we define + * it here. + * + * NOTE! This default implementation assumes that if the architecture + * support ioport mapping (HAS_IOPORT_MAP), the ioport mapping will + * be fixed to the range [ PCI_IOBASE, PCI_IOBASE+IO_SPACE_LIMIT [, + * and does not need unmapping with 'ioport_unmap()'. + * + * If you have different rules for your architecture, you need to + * implement your own pci_iounmap() that knows the rules for where + * and how IO vs MEM get mapped. + * + * This code is odd, and the ARCH_HAS/ARCH_WANTS #define logic comes + * from legacy <asm-generic/io.h> header file behavior. In particular, + * it would seem to make sense to do the iounmap(p) for the non-IO-space + * case here regardless, but that's not what the old header file code + * did. Probably incorrectly, but this is meant to be bug-for-bug + * compatible. + */ +#if defined(ARCH_WANTS_GENERIC_PCI_IOUNMAP) + +void pci_iounmap(struct pci_dev *dev, void __iomem *p) +{ +#ifdef ARCH_HAS_GENERIC_IOPORT_MAP + uintptr_t start = (uintptr_t) PCI_IOBASE; + uintptr_t addr = (uintptr_t) p; + + if (addr >= start && addr < start + IO_SPACE_LIMIT) + return; + iounmap(p); +#endif +} +EXPORT_SYMBOL(pci_iounmap); + +#endif /* ARCH_WANTS_GENERIC_PCI_IOUNMAP */ + #endif /* CONFIG_PCI */ diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 4f6c6ebbbbde..af9302141bcf 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -1,9 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only -#define pr_fmt(fmt) "%s: " fmt "\n", __func__ +#define pr_fmt(fmt) "%s: " fmt, __func__ #include <linux/kernel.h> #include <linux/sched.h> #include <linux/wait.h> +#include <linux/slab.h> +#include <linux/mm.h> #include <linux/percpu-refcount.h> /* @@ -50,9 +52,10 @@ static unsigned long __percpu *percpu_count_ptr(struct percpu_ref *ref) * @flags: PERCPU_REF_INIT_* flags * @gfp: allocation mask to use * - * Initializes @ref. If @flags is zero, @ref starts in percpu mode with a - * refcount of 1; analagous to atomic_long_set(ref, 1). See the - * definitions of PERCPU_REF_INIT_* flags for flag behaviors. + * Initializes @ref. @ref starts out in percpu mode with a refcount of 1 unless + * @flags contains PERCPU_REF_INIT_ATOMIC or PERCPU_REF_INIT_DEAD. These flags + * change the start state to atomic with the latter setting the initial refcount + * to 0. See the definitions of PERCPU_REF_INIT_* flags for flag behaviors. * * Note that @release must not sleep - it may potentially be called from RCU * callback context by percpu_ref_kill(). @@ -63,18 +66,25 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release, size_t align = max_t(size_t, 1 << __PERCPU_REF_FLAG_BITS, __alignof__(unsigned long)); unsigned long start_count = 0; + struct percpu_ref_data *data; ref->percpu_count_ptr = (unsigned long) __alloc_percpu_gfp(sizeof(unsigned long), align, gfp); if (!ref->percpu_count_ptr) return -ENOMEM; - ref->force_atomic = flags & PERCPU_REF_INIT_ATOMIC; - ref->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT; + data = kzalloc(sizeof(*ref->data), gfp); + if (!data) { + free_percpu((void __percpu *)ref->percpu_count_ptr); + return -ENOMEM; + } + + data->force_atomic = flags & PERCPU_REF_INIT_ATOMIC; + data->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT; if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) { ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC; - ref->allow_reinit = true; + data->allow_reinit = true; } else { start_count += PERCPU_COUNT_BIAS; } @@ -84,14 +94,28 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release, else start_count++; - atomic_long_set(&ref->count, start_count); + atomic_long_set(&data->count, start_count); - ref->release = release; - ref->confirm_switch = NULL; + data->release = release; + data->confirm_switch = NULL; + data->ref = ref; + ref->data = data; return 0; } EXPORT_SYMBOL_GPL(percpu_ref_init); +static void __percpu_ref_exit(struct percpu_ref *ref) +{ + unsigned long __percpu *percpu_count = percpu_count_ptr(ref); + + if (percpu_count) { + /* non-NULL confirm_switch indicates switching in progress */ + WARN_ON_ONCE(ref->data && ref->data->confirm_switch); + free_percpu(percpu_count); + ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD; + } +} + /** * percpu_ref_exit - undo percpu_ref_init() * @ref: percpu_ref to exit @@ -104,27 +128,36 @@ EXPORT_SYMBOL_GPL(percpu_ref_init); */ void percpu_ref_exit(struct percpu_ref *ref) { - unsigned long __percpu *percpu_count = percpu_count_ptr(ref); + struct percpu_ref_data *data = ref->data; + unsigned long flags; - if (percpu_count) { - /* non-NULL confirm_switch indicates switching in progress */ - WARN_ON_ONCE(ref->confirm_switch); - free_percpu(percpu_count); - ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD; - } + __percpu_ref_exit(ref); + + if (!data) + return; + + spin_lock_irqsave(&percpu_ref_switch_lock, flags); + ref->percpu_count_ptr |= atomic_long_read(&ref->data->count) << + __PERCPU_REF_FLAG_BITS; + ref->data = NULL; + spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); + + kfree(data); } EXPORT_SYMBOL_GPL(percpu_ref_exit); static void percpu_ref_call_confirm_rcu(struct rcu_head *rcu) { - struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu); + struct percpu_ref_data *data = container_of(rcu, + struct percpu_ref_data, rcu); + struct percpu_ref *ref = data->ref; - ref->confirm_switch(ref); - ref->confirm_switch = NULL; + data->confirm_switch(ref); + data->confirm_switch = NULL; wake_up_all(&percpu_ref_switch_waitq); - if (!ref->allow_reinit) - percpu_ref_exit(ref); + if (!data->allow_reinit) + __percpu_ref_exit(ref); /* drop ref from percpu_ref_switch_to_atomic() */ percpu_ref_put(ref); @@ -132,16 +165,19 @@ static void percpu_ref_call_confirm_rcu(struct rcu_head *rcu) static void percpu_ref_switch_to_atomic_rcu(struct rcu_head *rcu) { - struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu); + struct percpu_ref_data *data = container_of(rcu, + struct percpu_ref_data, rcu); + struct percpu_ref *ref = data->ref; unsigned long __percpu *percpu_count = percpu_count_ptr(ref); + static atomic_t underflows; unsigned long count = 0; int cpu; for_each_possible_cpu(cpu) count += *per_cpu_ptr(percpu_count, cpu); - pr_debug("global %ld percpu %ld", - atomic_long_read(&ref->count), (long)count); + pr_debug("global %lu percpu %lu\n", + atomic_long_read(&data->count), count); /* * It's crucial that we sum the percpu counters _before_ adding the sum @@ -155,11 +191,15 @@ static void percpu_ref_switch_to_atomic_rcu(struct rcu_head *rcu) * reaching 0 before we add the percpu counts. But doing it at the same * time is equivalent and saves us atomic operations: */ - atomic_long_add((long)count - PERCPU_COUNT_BIAS, &ref->count); - - WARN_ONCE(atomic_long_read(&ref->count) <= 0, - "percpu ref (%ps) <= 0 (%ld) after switching to atomic", - ref->release, atomic_long_read(&ref->count)); + atomic_long_add((long)count - PERCPU_COUNT_BIAS, &data->count); + + if (WARN_ONCE(atomic_long_read(&data->count) <= 0, + "percpu ref (%ps) <= 0 (%ld) after switching to atomic", + data->release, atomic_long_read(&data->count)) && + atomic_inc_return(&underflows) < 4) { + pr_err("%s(): percpu_ref underflow", __func__); + mem_dump_obj(data); + } /* @ref is viewed as dead on all CPUs, send out switch confirmation */ percpu_ref_call_confirm_rcu(rcu); @@ -185,10 +225,11 @@ static void __percpu_ref_switch_to_atomic(struct percpu_ref *ref, * Non-NULL ->confirm_switch is used to indicate that switching is * in progress. Use noop one if unspecified. */ - ref->confirm_switch = confirm_switch ?: percpu_ref_noop_confirm_switch; + ref->data->confirm_switch = confirm_switch ?: + percpu_ref_noop_confirm_switch; percpu_ref_get(ref); /* put after confirmation */ - call_rcu(&ref->rcu, percpu_ref_switch_to_atomic_rcu); + call_rcu(&ref->data->rcu, percpu_ref_switch_to_atomic_rcu); } static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref) @@ -201,10 +242,10 @@ static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref) if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC)) return; - if (WARN_ON_ONCE(!ref->allow_reinit)) + if (WARN_ON_ONCE(!ref->data->allow_reinit)) return; - atomic_long_add(PERCPU_COUNT_BIAS, &ref->count); + atomic_long_add(PERCPU_COUNT_BIAS, &ref->data->count); /* * Restore per-cpu operation. smp_store_release() is paired @@ -222,6 +263,8 @@ static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref) static void __percpu_ref_switch_mode(struct percpu_ref *ref, percpu_ref_func_t *confirm_switch) { + struct percpu_ref_data *data = ref->data; + lockdep_assert_held(&percpu_ref_switch_lock); /* @@ -229,10 +272,10 @@ static void __percpu_ref_switch_mode(struct percpu_ref *ref, * its completion. If the caller ensures that ATOMIC switching * isn't in progress, this function can be called from any context. */ - wait_event_lock_irq(percpu_ref_switch_waitq, !ref->confirm_switch, + wait_event_lock_irq(percpu_ref_switch_waitq, !data->confirm_switch, percpu_ref_switch_lock); - if (ref->force_atomic || (ref->percpu_count_ptr & __PERCPU_REF_DEAD)) + if (data->force_atomic || percpu_ref_is_dying(ref)) __percpu_ref_switch_to_atomic(ref, confirm_switch); else __percpu_ref_switch_to_percpu(ref); @@ -265,7 +308,7 @@ void percpu_ref_switch_to_atomic(struct percpu_ref *ref, spin_lock_irqsave(&percpu_ref_switch_lock, flags); - ref->force_atomic = true; + ref->data->force_atomic = true; __percpu_ref_switch_mode(ref, confirm_switch); spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); @@ -283,7 +326,7 @@ EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic); void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref) { percpu_ref_switch_to_atomic(ref, NULL); - wait_event(percpu_ref_switch_waitq, !ref->confirm_switch); + wait_event(percpu_ref_switch_waitq, !ref->data->confirm_switch); } EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic_sync); @@ -311,7 +354,7 @@ void percpu_ref_switch_to_percpu(struct percpu_ref *ref) spin_lock_irqsave(&percpu_ref_switch_lock, flags); - ref->force_atomic = false; + ref->data->force_atomic = false; __percpu_ref_switch_mode(ref, NULL); spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); @@ -342,8 +385,9 @@ void percpu_ref_kill_and_confirm(struct percpu_ref *ref, spin_lock_irqsave(&percpu_ref_switch_lock, flags); - WARN_ONCE(ref->percpu_count_ptr & __PERCPU_REF_DEAD, - "%s called more than once on %ps!", __func__, ref->release); + WARN_ONCE(percpu_ref_is_dying(ref), + "%s called more than once on %ps!", __func__, + ref->data->release); ref->percpu_count_ptr |= __PERCPU_REF_DEAD; __percpu_ref_switch_mode(ref, confirm_kill); @@ -354,6 +398,34 @@ void percpu_ref_kill_and_confirm(struct percpu_ref *ref, EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm); /** + * percpu_ref_is_zero - test whether a percpu refcount reached zero + * @ref: percpu_ref to test + * + * Returns %true if @ref reached zero. + * + * This function is safe to call as long as @ref is between init and exit. + */ +bool percpu_ref_is_zero(struct percpu_ref *ref) +{ + unsigned long __percpu *percpu_count; + unsigned long count, flags; + + if (__ref_is_percpu(ref, &percpu_count)) + return false; + + /* protect us from being destroyed */ + spin_lock_irqsave(&percpu_ref_switch_lock, flags); + if (ref->data) + count = atomic_long_read(&ref->data->count); + else + count = ref->percpu_count_ptr >> __PERCPU_REF_FLAG_BITS; + spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); + + return count == 0; +} +EXPORT_SYMBOL_GPL(percpu_ref_is_zero); + +/** * percpu_ref_reinit - re-initialize a percpu refcount * @ref: perpcu_ref to re-initialize * @@ -393,7 +465,7 @@ void percpu_ref_resurrect(struct percpu_ref *ref) spin_lock_irqsave(&percpu_ref_switch_lock, flags); - WARN_ON_ONCE(!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)); + WARN_ON_ONCE(!percpu_ref_is_dying(ref)); WARN_ON_ONCE(__ref_is_percpu(ref, &percpu_count)); ref->percpu_count_ptr &= ~__PERCPU_REF_DEAD; diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index a66595ba5543..ed610b75dc32 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c @@ -17,7 +17,7 @@ static DEFINE_SPINLOCK(percpu_counters_lock); #ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER -static struct debug_obj_descr percpu_counter_debug_descr; +static const struct debug_obj_descr percpu_counter_debug_descr; static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state) { @@ -33,7 +33,7 @@ static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state) } } -static struct debug_obj_descr percpu_counter_debug_descr = { +static const struct debug_obj_descr percpu_counter_debug_descr = { .name = "percpu_counter", .fixup_free = percpu_counter_fixup_free, }; @@ -72,7 +72,7 @@ void percpu_counter_set(struct percpu_counter *fbc, s64 amount) } EXPORT_SYMBOL(percpu_counter_set); -/** +/* * This function is both preempt and irq safe. The former is due to explicit * preemption disable. The latter is guaranteed by the fact that the slow path * is explicitly protected by an irq-safe spinlock whereas the fast patch uses @@ -85,7 +85,7 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch) preempt_disable(); count = __this_cpu_read(*fbc->counters) + amount; - if (count >= batch || count <= -batch) { + if (abs(count) >= batch) { unsigned long flags; raw_spin_lock_irqsave(&fbc->lock, flags); fbc->count += count; @@ -99,6 +99,25 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch) EXPORT_SYMBOL(percpu_counter_add_batch); /* + * For percpu_counter with a big batch, the devication of its count could + * be big, and there is requirement to reduce the deviation, like when the + * counter's batch could be runtime decreased to get a better accuracy, + * which can be achieved by running this sync function on each CPU. + */ +void percpu_counter_sync(struct percpu_counter *fbc) +{ + unsigned long flags; + s64 count; + + raw_spin_lock_irqsave(&fbc->lock, flags); + count = __this_cpu_read(*fbc->counters); + fbc->count += count; + __this_cpu_sub(*fbc->counters, count); + raw_spin_unlock_irqrestore(&fbc->lock, flags); +} +EXPORT_SYMBOL(percpu_counter_sync); + +/* * Add up all the per-cpu counts, return the result. This is a more accurate * but much slower version of percpu_counter_read_positive() */ diff --git a/lib/pldmfw/Makefile b/lib/pldmfw/Makefile new file mode 100644 index 000000000000..99ad10711abe --- /dev/null +++ b/lib/pldmfw/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_PLDMFW) += pldmfw.o diff --git a/lib/pldmfw/pldmfw.c b/lib/pldmfw/pldmfw.c new file mode 100644 index 000000000000..6e77eb6d8e72 --- /dev/null +++ b/lib/pldmfw/pldmfw.c @@ -0,0 +1,879 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2018-2019, Intel Corporation. */ + +#include <asm/unaligned.h> +#include <linux/crc32.h> +#include <linux/device.h> +#include <linux/firmware.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/pldmfw.h> +#include <linux/slab.h> +#include <linux/uuid.h> + +#include "pldmfw_private.h" + +/* Internal structure used to store details about the PLDM image file as it is + * being validated and processed. + */ +struct pldmfw_priv { + struct pldmfw *context; + const struct firmware *fw; + + /* current offset of firmware image */ + size_t offset; + + struct list_head records; + struct list_head components; + + /* PLDM Firmware Package Header */ + const struct __pldm_header *header; + u16 total_header_size; + + /* length of the component bitmap */ + u16 component_bitmap_len; + u16 bitmap_size; + + /* Start of the component image information */ + u16 component_count; + const u8 *component_start; + + /* Start pf the firmware device id records */ + const u8 *record_start; + u8 record_count; + + /* The CRC at the end of the package header */ + u32 header_crc; + + struct pldmfw_record *matching_record; +}; + +/** + * pldm_check_fw_space - Verify that the firmware image has space left + * @data: pointer to private data + * @offset: offset to start from + * @length: length to check for + * + * Verify that the firmware data can hold a chunk of bytes with the specified + * offset and length. + * + * Returns: zero on success, or -EFAULT if the image does not have enough + * space left to fit the expected length. + */ +static int +pldm_check_fw_space(struct pldmfw_priv *data, size_t offset, size_t length) +{ + size_t expected_size = offset + length; + struct device *dev = data->context->dev; + + if (data->fw->size < expected_size) { + dev_dbg(dev, "Firmware file size smaller than expected. Got %zu bytes, needed %zu bytes\n", + data->fw->size, expected_size); + return -EFAULT; + } + + return 0; +} + +/** + * pldm_move_fw_offset - Move the current firmware offset forward + * @data: pointer to private data + * @bytes_to_move: number of bytes to move the offset forward by + * + * Check that there is enough space past the current offset, and then move the + * offset forward by this amount. + * + * Returns: zero on success, or -EFAULT if the image is too small to fit the + * expected length. + */ +static int +pldm_move_fw_offset(struct pldmfw_priv *data, size_t bytes_to_move) +{ + int err; + + err = pldm_check_fw_space(data, data->offset, bytes_to_move); + if (err) + return err; + + data->offset += bytes_to_move; + + return 0; +} + +/** + * pldm_parse_header - Validate and extract details about the PLDM header + * @data: pointer to private data + * + * Performs initial basic verification of the PLDM image, up to the first + * firmware record. + * + * This includes the following checks and extractions + * + * * Verify that the UUID at the start of the header matches the expected + * value as defined in the DSP0267 PLDM specification + * * Check that the revision is 0x01 + * * Extract the total header_size and verify that the image is large enough + * to contain at least the length of this header + * * Extract the size of the component bitmap length + * * Extract a pointer to the start of the record area + * + * Returns: zero on success, or a negative error code on failure. + */ +static int pldm_parse_header(struct pldmfw_priv *data) +{ + const struct __pldmfw_record_area *record_area; + struct device *dev = data->context->dev; + const struct __pldm_header *header; + size_t header_size; + int err; + + err = pldm_move_fw_offset(data, sizeof(*header)); + if (err) + return err; + + header = (const struct __pldm_header *)data->fw->data; + data->header = header; + + if (!uuid_equal(&header->id, &pldm_firmware_header_id)) { + dev_dbg(dev, "Invalid package header identifier. Expected UUID %pUB, but got %pUB\n", + &pldm_firmware_header_id, &header->id); + return -EINVAL; + } + + if (header->revision != PACKAGE_HEADER_FORMAT_REVISION) { + dev_dbg(dev, "Invalid package header revision. Expected revision %u but got %u\n", + PACKAGE_HEADER_FORMAT_REVISION, header->revision); + return -EOPNOTSUPP; + } + + data->total_header_size = get_unaligned_le16(&header->size); + header_size = data->total_header_size - sizeof(*header); + + err = pldm_check_fw_space(data, data->offset, header_size); + if (err) + return err; + + data->component_bitmap_len = + get_unaligned_le16(&header->component_bitmap_len); + + if (data->component_bitmap_len % 8 != 0) { + dev_dbg(dev, "Invalid component bitmap length. The length is %u, which is not a multiple of 8\n", + data->component_bitmap_len); + return -EINVAL; + } + + data->bitmap_size = data->component_bitmap_len / 8; + + err = pldm_move_fw_offset(data, header->version_len); + if (err) + return err; + + /* extract a pointer to the record area, which just follows the main + * PLDM header data. + */ + record_area = (const struct __pldmfw_record_area *)(data->fw->data + + data->offset); + + err = pldm_move_fw_offset(data, sizeof(*record_area)); + if (err) + return err; + + data->record_count = record_area->record_count; + data->record_start = record_area->records; + + return 0; +} + +/** + * pldm_check_desc_tlv_len - Check that the length matches expectation + * @data: pointer to image details + * @type: the descriptor type + * @size: the length from the descriptor header + * + * If the descriptor type is one of the documented descriptor types according + * to the standard, verify that the provided length matches. + * + * If the type is not recognized or is VENDOR_DEFINED, return zero. + * + * Returns: zero on success, or -EINVAL if the specified size of a standard + * TLV does not match the expected value defined for that TLV. + */ +static int +pldm_check_desc_tlv_len(struct pldmfw_priv *data, u16 type, u16 size) +{ + struct device *dev = data->context->dev; + u16 expected_size; + + switch (type) { + case PLDM_DESC_ID_PCI_VENDOR_ID: + case PLDM_DESC_ID_PCI_DEVICE_ID: + case PLDM_DESC_ID_PCI_SUBVENDOR_ID: + case PLDM_DESC_ID_PCI_SUBDEV_ID: + expected_size = 2; + break; + case PLDM_DESC_ID_PCI_REVISION_ID: + expected_size = 1; + break; + case PLDM_DESC_ID_PNP_VENDOR_ID: + expected_size = 3; + break; + case PLDM_DESC_ID_IANA_ENTERPRISE_ID: + case PLDM_DESC_ID_ACPI_VENDOR_ID: + case PLDM_DESC_ID_PNP_PRODUCT_ID: + case PLDM_DESC_ID_ACPI_PRODUCT_ID: + expected_size = 4; + break; + case PLDM_DESC_ID_UUID: + expected_size = 16; + break; + case PLDM_DESC_ID_VENDOR_DEFINED: + return 0; + default: + /* Do not report an error on an unexpected TLV */ + dev_dbg(dev, "Found unrecognized TLV type 0x%04x\n", type); + return 0; + } + + if (size != expected_size) { + dev_dbg(dev, "Found TLV type 0x%04x with unexpected length. Got %u bytes, but expected %u bytes\n", + type, size, expected_size); + return -EINVAL; + } + + return 0; +} + +/** + * pldm_parse_desc_tlvs - Check and skip past a number of TLVs + * @data: pointer to private data + * @record: pointer to the record this TLV belongs too + * @desc_count: descriptor count + * + * From the current offset, read and extract the descriptor TLVs, updating the + * current offset each time. + * + * Returns: zero on success, or a negative error code on failure. + */ +static int +pldm_parse_desc_tlvs(struct pldmfw_priv *data, struct pldmfw_record *record, u8 desc_count) +{ + const struct __pldmfw_desc_tlv *__desc; + const u8 *desc_start; + u8 i; + + desc_start = data->fw->data + data->offset; + + pldm_for_each_desc_tlv(i, __desc, desc_start, desc_count) { + struct pldmfw_desc_tlv *desc; + int err; + u16 type, size; + + err = pldm_move_fw_offset(data, sizeof(*__desc)); + if (err) + return err; + + type = get_unaligned_le16(&__desc->type); + + /* According to DSP0267, this only includes the data field */ + size = get_unaligned_le16(&__desc->size); + + err = pldm_check_desc_tlv_len(data, type, size); + if (err) + return err; + + /* check that we have space and move the offset forward */ + err = pldm_move_fw_offset(data, size); + if (err) + return err; + + desc = kzalloc(sizeof(*desc), GFP_KERNEL); + if (!desc) + return -ENOMEM; + + desc->type = type; + desc->size = size; + desc->data = __desc->data; + + list_add_tail(&desc->entry, &record->descs); + } + + return 0; +} + +/** + * pldm_parse_one_record - Verify size of one PLDM record + * @data: pointer to image details + * @__record: pointer to the record to check + * + * This function checks that the record size does not exceed either the size + * of the firmware file or the total length specified in the header section. + * + * It also verifies that the recorded length of the start of the record + * matches the size calculated by adding the static structure length, the + * component bitmap length, the version string length, the length of all + * descriptor TLVs, and the length of the package data. + * + * Returns: zero on success, or a negative error code on failure. + */ +static int +pldm_parse_one_record(struct pldmfw_priv *data, + const struct __pldmfw_record_info *__record) +{ + struct pldmfw_record *record; + size_t measured_length; + int err; + const u8 *bitmap_ptr; + u16 record_len; + int i; + + /* Make a copy and insert it into the record list */ + record = kzalloc(sizeof(*record), GFP_KERNEL); + if (!record) + return -ENOMEM; + + INIT_LIST_HEAD(&record->descs); + list_add_tail(&record->entry, &data->records); + + /* Then check that we have space and move the offset */ + err = pldm_move_fw_offset(data, sizeof(*__record)); + if (err) + return err; + + record_len = get_unaligned_le16(&__record->record_len); + record->package_data_len = get_unaligned_le16(&__record->package_data_len); + record->version_len = __record->version_len; + record->version_type = __record->version_type; + + bitmap_ptr = data->fw->data + data->offset; + + /* check that we have space for the component bitmap length */ + err = pldm_move_fw_offset(data, data->bitmap_size); + if (err) + return err; + + record->component_bitmap_len = data->component_bitmap_len; + record->component_bitmap = bitmap_zalloc(record->component_bitmap_len, + GFP_KERNEL); + if (!record->component_bitmap) + return -ENOMEM; + + for (i = 0; i < data->bitmap_size; i++) + bitmap_set_value8(record->component_bitmap, bitmap_ptr[i], i * 8); + + record->version_string = data->fw->data + data->offset; + + err = pldm_move_fw_offset(data, record->version_len); + if (err) + return err; + + /* Scan through the descriptor TLVs and find the end */ + err = pldm_parse_desc_tlvs(data, record, __record->descriptor_count); + if (err) + return err; + + record->package_data = data->fw->data + data->offset; + + err = pldm_move_fw_offset(data, record->package_data_len); + if (err) + return err; + + measured_length = data->offset - ((const u8 *)__record - data->fw->data); + if (measured_length != record_len) { + dev_dbg(data->context->dev, "Unexpected record length. Measured record length is %zu bytes, expected length is %u bytes\n", + measured_length, record_len); + return -EFAULT; + } + + return 0; +} + +/** + * pldm_parse_records - Locate the start of the component area + * @data: pointer to private data + * + * Extract the record count, and loop through each record, searching for the + * component area. + * + * Returns: zero on success, or a negative error code on failure. + */ +static int pldm_parse_records(struct pldmfw_priv *data) +{ + const struct __pldmfw_component_area *component_area; + const struct __pldmfw_record_info *record; + int err; + u8 i; + + pldm_for_each_record(i, record, data->record_start, data->record_count) { + err = pldm_parse_one_record(data, record); + if (err) + return err; + } + + /* Extract a pointer to the component area, which just follows the + * PLDM device record data. + */ + component_area = (const struct __pldmfw_component_area *)(data->fw->data + data->offset); + + err = pldm_move_fw_offset(data, sizeof(*component_area)); + if (err) + return err; + + data->component_count = + get_unaligned_le16(&component_area->component_image_count); + data->component_start = component_area->components; + + return 0; +} + +/** + * pldm_parse_components - Locate the CRC header checksum + * @data: pointer to private data + * + * Extract the component count, and find the pointer to the component area. + * Scan through each component searching for the end, which should point to + * the package header checksum. + * + * Extract the package header CRC and save it for verification. + * + * Returns: zero on success, or a negative error code on failure. + */ +static int pldm_parse_components(struct pldmfw_priv *data) +{ + const struct __pldmfw_component_info *__component; + struct device *dev = data->context->dev; + const u8 *header_crc_ptr; + int err; + u8 i; + + pldm_for_each_component(i, __component, data->component_start, data->component_count) { + struct pldmfw_component *component; + u32 offset, size; + + err = pldm_move_fw_offset(data, sizeof(*__component)); + if (err) + return err; + + err = pldm_move_fw_offset(data, __component->version_len); + if (err) + return err; + + offset = get_unaligned_le32(&__component->location_offset); + size = get_unaligned_le32(&__component->size); + + err = pldm_check_fw_space(data, offset, size); + if (err) + return err; + + component = kzalloc(sizeof(*component), GFP_KERNEL); + if (!component) + return -ENOMEM; + + component->index = i; + component->classification = get_unaligned_le16(&__component->classification); + component->identifier = get_unaligned_le16(&__component->identifier); + component->comparison_stamp = get_unaligned_le32(&__component->comparison_stamp); + component->options = get_unaligned_le16(&__component->options); + component->activation_method = get_unaligned_le16(&__component->activation_method); + component->version_type = __component->version_type; + component->version_len = __component->version_len; + component->version_string = __component->version_string; + component->component_data = data->fw->data + offset; + component->component_size = size; + + list_add_tail(&component->entry, &data->components); + } + + header_crc_ptr = data->fw->data + data->offset; + + err = pldm_move_fw_offset(data, sizeof(data->header_crc)); + if (err) + return err; + + /* Make sure that we reached the expected offset */ + if (data->offset != data->total_header_size) { + dev_dbg(dev, "Invalid firmware header size. Expected %u but got %zu\n", + data->total_header_size, data->offset); + return -EFAULT; + } + + data->header_crc = get_unaligned_le32(header_crc_ptr); + + return 0; +} + +/** + * pldm_verify_header_crc - Verify that the CRC in the header matches + * @data: pointer to private data + * + * Calculates the 32-bit CRC using the standard IEEE 802.3 CRC polynomial and + * compares it to the value stored in the header. + * + * Returns: zero on success if the CRC matches, or -EBADMSG on an invalid CRC. + */ +static int pldm_verify_header_crc(struct pldmfw_priv *data) +{ + struct device *dev = data->context->dev; + u32 calculated_crc; + size_t length; + + /* Calculate the 32-bit CRC of the header header contents up to but + * not including the checksum. Note that the Linux crc32_le function + * does not perform an expected final XOR. + */ + length = data->offset - sizeof(data->header_crc); + calculated_crc = crc32_le(~0, data->fw->data, length) ^ ~0; + + if (calculated_crc != data->header_crc) { + dev_dbg(dev, "Invalid CRC in firmware header. Got 0x%08x but expected 0x%08x\n", + calculated_crc, data->header_crc); + return -EBADMSG; + } + + return 0; +} + +/** + * pldmfw_free_priv - Free memory allocated while parsing the PLDM image + * @data: pointer to the PLDM data structure + * + * Loops through and clears all allocated memory associated with each + * allocated descriptor, record, and component. + */ +static void pldmfw_free_priv(struct pldmfw_priv *data) +{ + struct pldmfw_component *component, *c_safe; + struct pldmfw_record *record, *r_safe; + struct pldmfw_desc_tlv *desc, *d_safe; + + list_for_each_entry_safe(component, c_safe, &data->components, entry) { + list_del(&component->entry); + kfree(component); + } + + list_for_each_entry_safe(record, r_safe, &data->records, entry) { + list_for_each_entry_safe(desc, d_safe, &record->descs, entry) { + list_del(&desc->entry); + kfree(desc); + } + + if (record->component_bitmap) { + bitmap_free(record->component_bitmap); + record->component_bitmap = NULL; + } + + list_del(&record->entry); + kfree(record); + } +} + +/** + * pldm_parse_image - parse and extract details from PLDM image + * @data: pointer to private data + * + * Verify that the firmware file contains valid data for a PLDM firmware + * file. Extract useful pointers and data from the firmware file and store + * them in the data structure. + * + * The PLDM firmware file format is defined in DMTF DSP0267 1.0.0. Care + * should be taken to use get_unaligned_le* when accessing data from the + * pointers in data. + * + * Returns: zero on success, or a negative error code on failure. + */ +static int pldm_parse_image(struct pldmfw_priv *data) +{ + int err; + + if (WARN_ON(!(data->context->dev && data->fw->data && data->fw->size))) + return -EINVAL; + + err = pldm_parse_header(data); + if (err) + return err; + + err = pldm_parse_records(data); + if (err) + return err; + + err = pldm_parse_components(data); + if (err) + return err; + + return pldm_verify_header_crc(data); +} + +/* these are u32 so that we can store PCI_ANY_ID */ +struct pldm_pci_record_id { + int vendor; + int device; + int subsystem_vendor; + int subsystem_device; +}; + +/** + * pldmfw_op_pci_match_record - Check if a PCI device matches the record + * @context: PLDM fw update structure + * @record: list of records extracted from the PLDM image + * + * Determine of the PCI device associated with this device matches the record + * data provided. + * + * Searches the descriptor TLVs and extracts the relevant descriptor data into + * a pldm_pci_record_id. This is then compared against the PCI device ID + * information. + * + * Returns: true if the device matches the record, false otherwise. + */ +bool pldmfw_op_pci_match_record(struct pldmfw *context, struct pldmfw_record *record) +{ + struct pci_dev *pdev = to_pci_dev(context->dev); + struct pldm_pci_record_id id = { + .vendor = PCI_ANY_ID, + .device = PCI_ANY_ID, + .subsystem_vendor = PCI_ANY_ID, + .subsystem_device = PCI_ANY_ID, + }; + struct pldmfw_desc_tlv *desc; + + list_for_each_entry(desc, &record->descs, entry) { + u16 value; + int *ptr; + + switch (desc->type) { + case PLDM_DESC_ID_PCI_VENDOR_ID: + ptr = &id.vendor; + break; + case PLDM_DESC_ID_PCI_DEVICE_ID: + ptr = &id.device; + break; + case PLDM_DESC_ID_PCI_SUBVENDOR_ID: + ptr = &id.subsystem_vendor; + break; + case PLDM_DESC_ID_PCI_SUBDEV_ID: + ptr = &id.subsystem_device; + break; + default: + /* Skip unrelated TLVs */ + continue; + } + + value = get_unaligned_le16(desc->data); + /* A value of zero for one of the descriptors is sometimes + * used when the record should ignore this field when matching + * device. For example if the record applies to any subsystem + * device or vendor. + */ + if (value) + *ptr = (int)value; + else + *ptr = PCI_ANY_ID; + } + + if ((id.vendor == PCI_ANY_ID || id.vendor == pdev->vendor) && + (id.device == PCI_ANY_ID || id.device == pdev->device) && + (id.subsystem_vendor == PCI_ANY_ID || id.subsystem_vendor == pdev->subsystem_vendor) && + (id.subsystem_device == PCI_ANY_ID || id.subsystem_device == pdev->subsystem_device)) + return true; + else + return false; +} +EXPORT_SYMBOL(pldmfw_op_pci_match_record); + +/** + * pldm_find_matching_record - Find the first matching PLDM record + * @data: pointer to private data + * + * Search through PLDM records and find the first matching entry. It is + * expected that only one entry matches. + * + * Store a pointer to the matching record, if found. + * + * Returns: zero on success, or -ENOENT if no matching record is found. + */ +static int pldm_find_matching_record(struct pldmfw_priv *data) +{ + struct pldmfw_record *record; + + list_for_each_entry(record, &data->records, entry) { + if (data->context->ops->match_record(data->context, record)) { + data->matching_record = record; + return 0; + } + } + + return -ENOENT; +} + +/** + * pldm_send_package_data - Send firmware the package data for the record + * @data: pointer to private data + * + * Send the package data associated with the matching record to the firmware, + * using the send_pkg_data operation. + * + * Returns: zero on success, or a negative error code on failure. + */ +static int +pldm_send_package_data(struct pldmfw_priv *data) +{ + struct pldmfw_record *record = data->matching_record; + const struct pldmfw_ops *ops = data->context->ops; + + return ops->send_package_data(data->context, record->package_data, + record->package_data_len); +} + +/** + * pldm_send_component_tables - Send component table information to firmware + * @data: pointer to private data + * + * Loop over each component, sending the applicable components to the firmware + * via the send_component_table operation. + * + * Returns: zero on success, or a negative error code on failure. + */ +static int +pldm_send_component_tables(struct pldmfw_priv *data) +{ + unsigned long *bitmap = data->matching_record->component_bitmap; + struct pldmfw_component *component; + int err; + + list_for_each_entry(component, &data->components, entry) { + u8 index = component->index, transfer_flag = 0; + + /* Skip components which are not intended for this device */ + if (!test_bit(index, bitmap)) + continue; + + /* determine whether this is the start, middle, end, or both + * the start and end of the component tables + */ + if (index == find_first_bit(bitmap, data->component_bitmap_len)) + transfer_flag |= PLDM_TRANSFER_FLAG_START; + if (index == find_last_bit(bitmap, data->component_bitmap_len)) + transfer_flag |= PLDM_TRANSFER_FLAG_END; + if (!transfer_flag) + transfer_flag = PLDM_TRANSFER_FLAG_MIDDLE; + + err = data->context->ops->send_component_table(data->context, + component, + transfer_flag); + if (err) + return err; + } + + return 0; +} + +/** + * pldm_flash_components - Program each component to device flash + * @data: pointer to private data + * + * Loop through each component that is active for the matching device record, + * and send it to the device driver for flashing. + * + * Returns: zero on success, or a negative error code on failure. + */ +static int pldm_flash_components(struct pldmfw_priv *data) +{ + unsigned long *bitmap = data->matching_record->component_bitmap; + struct pldmfw_component *component; + int err; + + list_for_each_entry(component, &data->components, entry) { + u8 index = component->index; + + /* Skip components which are not intended for this device */ + if (!test_bit(index, bitmap)) + continue; + + err = data->context->ops->flash_component(data->context, component); + if (err) + return err; + } + + return 0; +} + +/** + * pldm_finalize_update - Finalize the device flash update + * @data: pointer to private data + * + * Tell the device driver to perform any remaining logic to complete the + * device update. + * + * Returns: zero on success, or a PLFM_FWU error indicating the reason for + * failure. + */ +static int pldm_finalize_update(struct pldmfw_priv *data) +{ + if (data->context->ops->finalize_update) + return data->context->ops->finalize_update(data->context); + + return 0; +} + +/** + * pldmfw_flash_image - Write a PLDM-formatted firmware image to the device + * @context: ops and data for firmware update + * @fw: firmware object pointing to the relevant firmware file to program + * + * Parse the data for a given firmware file, verifying that it is a valid PLDM + * formatted image that matches this device. + * + * Extract the device record Package Data and Component Tables and send them + * to the device firmware. Extract and write the flash data for each of the + * components indicated in the firmware file. + * + * Returns: zero on success, or a negative error code on failure. + */ +int pldmfw_flash_image(struct pldmfw *context, const struct firmware *fw) +{ + struct pldmfw_priv *data; + int err; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + INIT_LIST_HEAD(&data->records); + INIT_LIST_HEAD(&data->components); + + data->fw = fw; + data->context = context; + + err = pldm_parse_image(data); + if (err) + goto out_release_data; + + err = pldm_find_matching_record(data); + if (err) + goto out_release_data; + + err = pldm_send_package_data(data); + if (err) + goto out_release_data; + + err = pldm_send_component_tables(data); + if (err) + goto out_release_data; + + err = pldm_flash_components(data); + if (err) + goto out_release_data; + + err = pldm_finalize_update(data); + +out_release_data: + pldmfw_free_priv(data); + kfree(data); + + return err; +} +EXPORT_SYMBOL(pldmfw_flash_image); + +MODULE_AUTHOR("Jacob Keller <jacob.e.keller@intel.com>"); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("PLDM firmware flash update library"); diff --git a/lib/pldmfw/pldmfw_private.h b/lib/pldmfw/pldmfw_private.h new file mode 100644 index 000000000000..687ef2200692 --- /dev/null +++ b/lib/pldmfw/pldmfw_private.h @@ -0,0 +1,238 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2018-2019, Intel Corporation. */ + +#ifndef _PLDMFW_PRIVATE_H_ +#define _PLDMFW_PRIVATE_H_ + +/* The following data structures define the layout of a firmware binary + * following the "PLDM For Firmware Update Specification", DMTF standard + * #DSP0267. + * + * pldmfw.c uses these structures to implement a simple engine that will parse + * a fw binary file in this format and perform a firmware update for a given + * device. + * + * Due to the variable sized data layout, alignment of fields within these + * structures is not guaranteed when reading. For this reason, all multi-byte + * field accesses should be done using the unaligned access macros. + * Additionally, the standard specifies that multi-byte fields are in + * LittleEndian format. + * + * The structure definitions are not made public, in order to keep direct + * accesses within code that is prepared to deal with the limitation of + * unaligned access. + */ + +/* UUID for PLDM firmware packages: f018878c-cb7d-4943-9800-a02f059aca02 */ +static const uuid_t pldm_firmware_header_id = + UUID_INIT(0xf018878c, 0xcb7d, 0x4943, + 0x98, 0x00, 0xa0, 0x2f, 0x05, 0x9a, 0xca, 0x02); + +/* Revision number of the PLDM header format this code supports */ +#define PACKAGE_HEADER_FORMAT_REVISION 0x01 + +/* timestamp104 structure defined in PLDM Base specification */ +#define PLDM_TIMESTAMP_SIZE 13 +struct __pldm_timestamp { + u8 b[PLDM_TIMESTAMP_SIZE]; +} __packed __aligned(1); + +/* Package Header Information */ +struct __pldm_header { + uuid_t id; /* PackageHeaderIdentifier */ + u8 revision; /* PackageHeaderFormatRevision */ + __le16 size; /* PackageHeaderSize */ + struct __pldm_timestamp release_date; /* PackageReleaseDateTime */ + __le16 component_bitmap_len; /* ComponentBitmapBitLength */ + u8 version_type; /* PackageVersionStringType */ + u8 version_len; /* PackageVersionStringLength */ + + /* + * DSP0267 also includes the following variable length fields at the + * end of this structure: + * + * PackageVersionString, length is version_len. + * + * The total size of this section is + * sizeof(pldm_header) + version_len; + */ + u8 version_string[]; /* PackageVersionString */ +} __packed __aligned(1); + +/* Firmware Device ID Record */ +struct __pldmfw_record_info { + __le16 record_len; /* RecordLength */ + u8 descriptor_count; /* DescriptorCount */ + __le32 device_update_flags; /* DeviceUpdateOptionFlags */ + u8 version_type; /* ComponentImageSetVersionType */ + u8 version_len; /* ComponentImageSetVersionLength */ + __le16 package_data_len; /* FirmwareDevicePackageDataLength */ + + /* + * DSP0267 also includes the following variable length fields at the + * end of this structure: + * + * ApplicableComponents, length is component_bitmap_len from header + * ComponentImageSetVersionString, length is version_len + * RecordDescriptors, a series of TLVs with 16bit type and length + * FirmwareDevicePackageData, length is package_data_len + * + * The total size of each record is + * sizeof(pldmfw_record_info) + + * component_bitmap_len (converted to bytes!) + + * version_len + + * <length of RecordDescriptors> + + * package_data_len + */ + u8 variable_record_data[]; +} __packed __aligned(1); + +/* Firmware Descriptor Definition */ +struct __pldmfw_desc_tlv { + __le16 type; /* DescriptorType */ + __le16 size; /* DescriptorSize */ + u8 data[]; /* DescriptorData */ +} __aligned(1); + +/* Firmware Device Identification Area */ +struct __pldmfw_record_area { + u8 record_count; /* DeviceIDRecordCount */ + /* This is not a struct type because the size of each record varies */ + u8 records[]; +} __aligned(1); + +/* Individual Component Image Information */ +struct __pldmfw_component_info { + __le16 classification; /* ComponentClassfication */ + __le16 identifier; /* ComponentIdentifier */ + __le32 comparison_stamp; /* ComponentComparisonStamp */ + __le16 options; /* componentOptions */ + __le16 activation_method; /* RequestedComponentActivationMethod */ + __le32 location_offset; /* ComponentLocationOffset */ + __le32 size; /* ComponentSize */ + u8 version_type; /* ComponentVersionStringType */ + u8 version_len; /* ComponentVersionStringLength */ + + /* + * DSP0267 also includes the following variable length fields at the + * end of this structure: + * + * ComponentVersionString, length is version_len + * + * The total size of this section is + * sizeof(pldmfw_component_info) + version_len; + */ + u8 version_string[]; /* ComponentVersionString */ +} __packed __aligned(1); + +/* Component Image Information Area */ +struct __pldmfw_component_area { + __le16 component_image_count; + /* This is not a struct type because the component size varies */ + u8 components[]; +} __aligned(1); + +/** + * pldm_first_desc_tlv + * @start: byte offset of the start of the descriptor TLVs + * + * Converts the starting offset of the descriptor TLVs into a pointer to the + * first descriptor. + */ +#define pldm_first_desc_tlv(start) \ + ((const struct __pldmfw_desc_tlv *)(start)) + +/** + * pldm_next_desc_tlv + * @desc: pointer to a descriptor TLV + * + * Finds the pointer to the next descriptor following a given descriptor + */ +#define pldm_next_desc_tlv(desc) \ + ((const struct __pldmfw_desc_tlv *)((desc)->data + \ + get_unaligned_le16(&(desc)->size))) + +/** + * pldm_for_each_desc_tlv + * @i: variable to store descriptor index + * @desc: variable to store descriptor pointer + * @start: byte offset of the start of the descriptors + * @count: the number of descriptors + * + * for loop macro to iterate over all of the descriptors of a given PLDM + * record. + */ +#define pldm_for_each_desc_tlv(i, desc, start, count) \ + for ((i) = 0, (desc) = pldm_first_desc_tlv(start); \ + (i) < (count); \ + (i)++, (desc) = pldm_next_desc_tlv(desc)) + +/** + * pldm_first_record + * @start: byte offset of the start of the PLDM records + * + * Converts a starting offset of the PLDM records into a pointer to the first + * record. + */ +#define pldm_first_record(start) \ + ((const struct __pldmfw_record_info *)(start)) + +/** + * pldm_next_record + * @record: pointer to a PLDM record + * + * Finds a pointer to the next record following a given record + */ +#define pldm_next_record(record) \ + ((const struct __pldmfw_record_info *) \ + ((const u8 *)(record) + get_unaligned_le16(&(record)->record_len))) + +/** + * pldm_for_each_record + * @i: variable to store record index + * @record: variable to store record pointer + * @start: byte offset of the start of the records + * @count: the number of records + * + * for loop macro to iterate over all of the records of a PLDM file. + */ +#define pldm_for_each_record(i, record, start, count) \ + for ((i) = 0, (record) = pldm_first_record(start); \ + (i) < (count); \ + (i)++, (record) = pldm_next_record(record)) + +/** + * pldm_first_component + * @start: byte offset of the start of the PLDM components + * + * Convert a starting offset of the PLDM components into a pointer to the + * first component + */ +#define pldm_first_component(start) \ + ((const struct __pldmfw_component_info *)(start)) + +/** + * pldm_next_component + * @component: pointer to a PLDM component + * + * Finds a pointer to the next component following a given component + */ +#define pldm_next_component(component) \ + ((const struct __pldmfw_component_info *)((component)->version_string + \ + (component)->version_len)) + +/** + * pldm_for_each_component + * @i: variable to store component index + * @component: variable to store component pointer + * @start: byte offset to the start of the first component + * @count: the number of components + * + * for loop macro to iterate over all of the components of a PLDM file. + */ +#define pldm_for_each_component(i, component, start, count) \ + for ((i) = 0, (component) = pldm_first_component(start); \ + (i) < (count); \ + (i)++, (component) = pldm_next_component(component)) + +#endif diff --git a/lib/radix-tree.c b/lib/radix-tree.c index c8fa1d274530..b3afafe46fff 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -27,7 +27,6 @@ #include <linux/string.h> #include <linux/xarray.h> - /* * Radix tree node cache. */ @@ -56,22 +55,12 @@ struct kmem_cache *radix_tree_node_cachep; #define IDR_PRELOAD_SIZE (IDR_MAX_PATH * 2 - 1) /* - * The IDA is even shorter since it uses a bitmap at the last level. - */ -#define IDA_INDEX_BITS (8 * sizeof(int) - 1 - ilog2(IDA_BITMAP_BITS)) -#define IDA_MAX_PATH (DIV_ROUND_UP(IDA_INDEX_BITS, \ - RADIX_TREE_MAP_SHIFT)) -#define IDA_PRELOAD_SIZE (IDA_MAX_PATH * 2 - 1) - -/* * Per-cpu pool of preloaded nodes */ -struct radix_tree_preload { - unsigned nr; - /* nodes->parent points to next preallocated node */ - struct radix_tree_node *nodes; +DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { + .lock = INIT_LOCAL_LOCK(lock), }; -static DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, }; +EXPORT_PER_CPU_SYMBOL_GPL(radix_tree_preloads); static inline struct radix_tree_node *entry_to_node(void *ptr) { @@ -177,9 +166,9 @@ static inline void all_tag_set(struct radix_tree_node *node, unsigned int tag) /** * radix_tree_find_next_bit - find the next set bit in a memory region * - * @addr: The address to base the search on - * @size: The bitmap size in bits - * @offset: The bitnumber to start searching at + * @node: where to begin the search + * @tag: the tag index + * @offset: the bitnumber to start searching at * * Unrollable variant of find_next_bit() for constant size arrays. * Tail bits starting from size to roundup(size, BITS_PER_LONG) must be zero. @@ -335,19 +324,19 @@ static __must_check int __radix_tree_preload(gfp_t gfp_mask, unsigned nr) int ret = -ENOMEM; /* - * Nodes preloaded by one cgroup can be be used by another cgroup, so + * Nodes preloaded by one cgroup can be used by another cgroup, so * they should never be accounted to any particular memory cgroup. */ gfp_mask &= ~__GFP_ACCOUNT; - preempt_disable(); + local_lock(&radix_tree_preloads.lock); rtp = this_cpu_ptr(&radix_tree_preloads); while (rtp->nr < nr) { - preempt_enable(); + local_unlock(&radix_tree_preloads.lock); node = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask); if (node == NULL) goto out; - preempt_disable(); + local_lock(&radix_tree_preloads.lock); rtp = this_cpu_ptr(&radix_tree_preloads); if (rtp->nr < nr) { node->parent = rtp->nodes; @@ -389,7 +378,7 @@ int radix_tree_maybe_preload(gfp_t gfp_mask) if (gfpflags_allow_blocking(gfp_mask)) return __radix_tree_preload(gfp_mask, RADIX_TREE_PRELOAD_SIZE); /* Preloading doesn't help anything with this gfp mask, skip it */ - preempt_disable(); + local_lock(&radix_tree_preloads.lock); return 0; } EXPORT_SYMBOL(radix_tree_maybe_preload); @@ -472,7 +461,7 @@ out: /** * radix_tree_shrink - shrink radix tree to minimum height - * @root radix tree root + * @root: radix tree root */ static inline bool radix_tree_shrink(struct radix_tree_root *root) { @@ -702,7 +691,7 @@ static inline int insert_entries(struct radix_tree_node *node, } /** - * __radix_tree_insert - insert into a radix tree + * radix_tree_insert - insert into a radix tree * @root: radix tree root * @index: index key * @item: item to insert @@ -930,6 +919,7 @@ EXPORT_SYMBOL(radix_tree_replace_slot); /** * radix_tree_iter_replace - replace item in a slot * @root: radix tree root + * @iter: iterator state * @slot: pointer to slot * @item: new item to store in the slot. * @@ -1039,7 +1029,7 @@ void *radix_tree_tag_clear(struct radix_tree_root *root, { struct radix_tree_node *node, *parent; unsigned long maxindex; - int uninitialized_var(offset); + int offset; radix_tree_load_root(root, &node, &maxindex); if (index > maxindex) @@ -1478,7 +1468,7 @@ EXPORT_SYMBOL(radix_tree_tagged); void idr_preload(gfp_t gfp_mask) { if (__radix_tree_preload(gfp_mask, IDR_PRELOAD_SIZE)) - preempt_disable(); + local_lock(&radix_tree_preloads.lock); } EXPORT_SYMBOL(idr_preload); diff --git a/lib/raid6/.gitignore b/lib/raid6/.gitignore index 3de0d8921286..6be57745afd1 100644 --- a/lib/raid6/.gitignore +++ b/lib/raid6/.gitignore @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only mktables altivec*.c int*.c diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile index 0083b5cc646c..45e17619422b 100644 --- a/lib/raid6/Makefile +++ b/lib/raid6/Makefile @@ -10,10 +10,12 @@ raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \ raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o -hostprogs-y += mktables +hostprogs += mktables ifeq ($(CONFIG_ALTIVEC),y) altivec_flags := -maltivec $(call cc-option,-mabi=altivec) +# Enable <altivec.h> +altivec_flags += -isystem $(shell $(CC) -print-file-name=include) ifdef CONFIG_CC_IS_CLANG # clang ppc port does not yet support -maltivec when -msoft-float is @@ -34,6 +36,8 @@ endif # ARM/NEON intrinsics in a non C99-compliant environment (such as the kernel) ifeq ($(CONFIG_KERNEL_MODE_NEON),y) NEON_FLAGS := -ffreestanding +# Enable <arm_neon.h> +NEON_FLAGS += -isystem $(shell $(CC) -print-file-name=include) ifeq ($(ARCH),arm) NEON_FLAGS += -march=armv7-a -mfloat-abi=softfp -mfpu=neon endif @@ -48,7 +52,7 @@ endif endif quiet_cmd_unroll = UNROLL $@ - cmd_unroll = $(AWK) -f$(srctree)/$(src)/unroll.awk -vN=$* < $< > $@ + cmd_unroll = $(AWK) -v N=$* -f $(srctree)/$(src)/unroll.awk < $< > $@ targets += int1.c int2.c int4.c int8.c int16.c int32.c $(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index 17417eee0866..6d5e5000fdd7 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -34,10 +34,8 @@ const struct raid6_calls * const raid6_algos[] = { &raid6_avx512x2, &raid6_avx512x1, #endif -#ifdef CONFIG_AS_AVX2 &raid6_avx2x2, &raid6_avx2x1, -#endif &raid6_sse2x2, &raid6_sse2x1, &raid6_sse1x2, @@ -51,11 +49,9 @@ const struct raid6_calls * const raid6_algos[] = { &raid6_avx512x2, &raid6_avx512x1, #endif -#ifdef CONFIG_AS_AVX2 &raid6_avx2x4, &raid6_avx2x2, &raid6_avx2x1, -#endif &raid6_sse2x4, &raid6_sse2x2, &raid6_sse2x1, @@ -97,13 +93,11 @@ void (*raid6_datap_recov)(int, size_t, int, void **); EXPORT_SYMBOL_GPL(raid6_datap_recov); const struct raid6_recov_calls *const raid6_recov_algos[] = { +#ifdef CONFIG_X86 #ifdef CONFIG_AS_AVX512 &raid6_recov_avx512, #endif -#ifdef CONFIG_AS_AVX2 &raid6_recov_avx2, -#endif -#ifdef CONFIG_AS_SSSE3 &raid6_recov_ssse3, #endif #ifdef CONFIG_S390 @@ -124,6 +118,9 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = { #define time_before(x, y) ((x) < (y)) #endif +#define RAID6_TEST_DISKS 8 +#define RAID6_TEST_DISKS_ORDER 3 + static inline const struct raid6_recov_calls *raid6_choose_recov(void) { const struct raid6_recov_calls *const *algo; @@ -146,7 +143,7 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) } static inline const struct raid6_calls *raid6_choose_gen( - void *(*const dptrs)[(65536/PAGE_SIZE)+2], const int disks) + void *(*const dptrs)[RAID6_TEST_DISKS], const int disks) { unsigned long perf, bestgenperf, bestxorperf, j0, j1; int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */ @@ -181,7 +178,8 @@ static inline const struct raid6_calls *raid6_choose_gen( best = *algo; } pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name, - (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2)); + (perf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); if (!(*algo)->xor_syndrome) continue; @@ -204,17 +202,24 @@ static inline const struct raid6_calls *raid6_choose_gen( bestxorperf = perf; pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name, - (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1)); + (perf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); } } if (best) { - pr_info("raid6: using algorithm %s gen() %ld MB/s\n", - best->name, - (bestgenperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2)); - if (best->xor_syndrome) - pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", - (bestxorperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1)); + if (IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { + pr_info("raid6: using algorithm %s gen() %ld MB/s\n", + best->name, + (bestgenperf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT+RAID6_TIME_JIFFIES_LG2)); + if (best->xor_syndrome) + pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", + (bestxorperf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); + } else + pr_info("raid6: skip pq benchmark and using algorithm %s\n", + best->name); raid6_call = *best; } else pr_err("raid6: Yikes! No algorithm found!\n"); @@ -228,27 +233,33 @@ static inline const struct raid6_calls *raid6_choose_gen( int __init raid6_select_algo(void) { - const int disks = (65536/PAGE_SIZE)+2; + const int disks = RAID6_TEST_DISKS; const struct raid6_calls *gen_best; const struct raid6_recov_calls *rec_best; - char *syndromes; - void *dptrs[(65536/PAGE_SIZE)+2]; - int i; - - for (i = 0; i < disks-2; i++) - dptrs[i] = ((char *)raid6_gfmul) + PAGE_SIZE*i; + char *disk_ptr, *p; + void *dptrs[RAID6_TEST_DISKS]; + int i, cycle; - /* Normal code - use a 2-page allocation to avoid D$ conflict */ - syndromes = (void *) __get_free_pages(GFP_KERNEL, 1); - - if (!syndromes) { + /* prepare the buffer and fill it circularly with gfmul table */ + disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); + if (!disk_ptr) { pr_err("raid6: Yikes! No memory available.\n"); return -ENOMEM; } - dptrs[disks-2] = syndromes; - dptrs[disks-1] = syndromes + PAGE_SIZE; + p = disk_ptr; + for (i = 0; i < disks; i++) + dptrs[i] = p + PAGE_SIZE * i; + + cycle = ((disks - 2) * PAGE_SIZE) / 65536; + for (i = 0; i < cycle; i++) { + memcpy(p, raid6_gfmul, 65536); + p += 65536; + } + + if ((disks - 2) * PAGE_SIZE % 65536) + memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); /* select raid gen_syndrome function */ gen_best = raid6_choose_gen(&dptrs, disks); @@ -256,7 +267,7 @@ int __init raid6_select_algo(void) /* select raid recover functions */ rec_best = raid6_choose_recov(); - free_pages((unsigned long)syndromes, 1); + free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); return gen_best && rec_best ? 0 : -EINVAL; } diff --git a/lib/raid6/avx2.c b/lib/raid6/avx2.c index 87184b6da28a..f299476e1d76 100644 --- a/lib/raid6/avx2.c +++ b/lib/raid6/avx2.c @@ -13,8 +13,6 @@ * */ -#ifdef CONFIG_AS_AVX2 - #include <linux/raid/pq.h> #include "x86.h" @@ -470,5 +468,3 @@ const struct raid6_calls raid6_avx2x4 = { 1 /* Has cache hints */ }; #endif - -#endif /* CONFIG_AS_AVX2 */ diff --git a/lib/raid6/mktables.c b/lib/raid6/mktables.c index 9c485df1308f..f02e10fa6238 100644 --- a/lib/raid6/mktables.c +++ b/lib/raid6/mktables.c @@ -56,8 +56,8 @@ int main(int argc, char *argv[]) uint8_t v; uint8_t exptbl[256], invtbl[256]; - printf("#include <linux/raid/pq.h>\n"); printf("#include <linux/export.h>\n"); + printf("#include <linux/raid/pq.h>\n"); /* Compute multiplication table */ printf("\nconst u8 __attribute__((aligned(256)))\n" diff --git a/lib/raid6/recov_avx2.c b/lib/raid6/recov_avx2.c index 7a3b5e7f66ee..4e8095403ee2 100644 --- a/lib/raid6/recov_avx2.c +++ b/lib/raid6/recov_avx2.c @@ -4,8 +4,6 @@ * Author: Jim Kukunas <james.t.kukunas@linux.intel.com> */ -#ifdef CONFIG_AS_AVX2 - #include <linux/raid/pq.h> #include "x86.h" @@ -313,7 +311,3 @@ const struct raid6_recov_calls raid6_recov_avx2 = { #endif .priority = 2, }; - -#else -#warning "your version of binutils lacks AVX2 support" -#endif diff --git a/lib/raid6/recov_ssse3.c b/lib/raid6/recov_ssse3.c index 1de97d2405d0..4bfa3c6b60de 100644 --- a/lib/raid6/recov_ssse3.c +++ b/lib/raid6/recov_ssse3.c @@ -3,8 +3,6 @@ * Copyright (C) 2012 Intel Corporation */ -#ifdef CONFIG_AS_SSSE3 - #include <linux/raid/pq.h> #include "x86.h" @@ -328,7 +326,3 @@ const struct raid6_recov_calls raid6_recov_ssse3 = { #endif .priority = 1, }; - -#else -#warning "your version of binutils lacks SSSE3 support" -#endif diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile index 3ab8720aa2f8..a4c7cd74cff5 100644 --- a/lib/raid6/test/Makefile +++ b/lib/raid6/test/Makefile @@ -34,14 +34,9 @@ endif ifeq ($(IS_X86),yes) OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o - CFLAGS += $(shell echo "pshufb %xmm0, %xmm0" | \ - gcc -c -x assembler - >&/dev/null && \ - rm ./-.o && echo -DCONFIG_AS_SSSE3=1) - CFLAGS += $(shell echo "vpbroadcastb %xmm0, %ymm1" | \ - gcc -c -x assembler - >&/dev/null && \ - rm ./-.o && echo -DCONFIG_AS_AVX2=1) + CFLAGS += -DCONFIG_X86 CFLAGS += $(shell echo "vpmovm2b %k1, %zmm5" | \ - gcc -c -x assembler - >&/dev/null && \ + gcc -c -x assembler - >/dev/null 2>&1 && \ rm ./-.o && echo -DCONFIG_AS_AVX512=1) else ifeq ($(HAS_NEON),yes) OBJS += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o diff --git a/lib/random32.c b/lib/random32.c index 763b920a6206..a57a0e18819d 100644 --- a/lib/random32.c +++ b/lib/random32.c @@ -38,17 +38,10 @@ #include <linux/jiffies.h> #include <linux/random.h> #include <linux/sched.h> +#include <linux/bitops.h> +#include <linux/slab.h> #include <asm/unaligned.h> - -#ifdef CONFIG_RANDOM32_SELFTEST -static void __init prandom_state_selftest(void); -#else -static inline void prandom_state_selftest(void) -{ -} -#endif - -static DEFINE_PER_CPU(struct rnd_state, net_rand_state) __latent_entropy; +#include <trace/events/random.h> /** * prandom_u32_state - seeded pseudo-random number generator. @@ -70,25 +63,6 @@ u32 prandom_u32_state(struct rnd_state *state) EXPORT_SYMBOL(prandom_u32_state); /** - * prandom_u32 - pseudo random number generator - * - * A 32 bit pseudo-random number is generated using a fast - * algorithm suitable for simulation. This algorithm is NOT - * considered safe for cryptographic use. - */ -u32 prandom_u32(void) -{ - struct rnd_state *state = &get_cpu_var(net_rand_state); - u32 res; - - res = prandom_u32_state(state); - put_cpu_var(net_rand_state); - - return res; -} -EXPORT_SYMBOL(prandom_u32); - -/** * prandom_bytes_state - get the requested number of pseudo-random bytes * * @state: pointer to state structure holding seeded state. @@ -119,20 +93,6 @@ void prandom_bytes_state(struct rnd_state *state, void *buf, size_t bytes) } EXPORT_SYMBOL(prandom_bytes_state); -/** - * prandom_bytes - get the requested number of pseudo-random bytes - * @buf: where to copy the pseudo-random bytes to - * @bytes: the requested number of bytes - */ -void prandom_bytes(void *buf, size_t bytes) -{ - struct rnd_state *state = &get_cpu_var(net_rand_state); - - prandom_bytes_state(state, buf, bytes); - put_cpu_var(net_rand_state); -} -EXPORT_SYMBOL(prandom_bytes); - static void prandom_warmup(struct rnd_state *state) { /* Calling RNG ten times to satisfy recurrence condition */ @@ -148,96 +108,6 @@ static void prandom_warmup(struct rnd_state *state) prandom_u32_state(state); } -static u32 __extract_hwseed(void) -{ - unsigned int val = 0; - - (void)(arch_get_random_seed_int(&val) || - arch_get_random_int(&val)); - - return val; -} - -static void prandom_seed_early(struct rnd_state *state, u32 seed, - bool mix_with_hwseed) -{ -#define LCG(x) ((x) * 69069U) /* super-duper LCG */ -#define HWSEED() (mix_with_hwseed ? __extract_hwseed() : 0) - state->s1 = __seed(HWSEED() ^ LCG(seed), 2U); - state->s2 = __seed(HWSEED() ^ LCG(state->s1), 8U); - state->s3 = __seed(HWSEED() ^ LCG(state->s2), 16U); - state->s4 = __seed(HWSEED() ^ LCG(state->s3), 128U); -} - -/** - * prandom_seed - add entropy to pseudo random number generator - * @entropy: entropy value - * - * Add some additional entropy to the prandom pool. - */ -void prandom_seed(u32 entropy) -{ - int i; - /* - * No locking on the CPUs, but then somewhat random results are, well, - * expected. - */ - for_each_possible_cpu(i) { - struct rnd_state *state = &per_cpu(net_rand_state, i); - - state->s1 = __seed(state->s1 ^ entropy, 2U); - prandom_warmup(state); - } -} -EXPORT_SYMBOL(prandom_seed); - -/* - * Generate some initially weak seeding values to allow - * to start the prandom_u32() engine. - */ -static int __init prandom_init(void) -{ - int i; - - prandom_state_selftest(); - - for_each_possible_cpu(i) { - struct rnd_state *state = &per_cpu(net_rand_state, i); - u32 weak_seed = (i + jiffies) ^ random_get_entropy(); - - prandom_seed_early(state, weak_seed, true); - prandom_warmup(state); - } - - return 0; -} -core_initcall(prandom_init); - -static void __prandom_timer(struct timer_list *unused); - -static DEFINE_TIMER(seed_timer, __prandom_timer); - -static void __prandom_timer(struct timer_list *unused) -{ - u32 entropy; - unsigned long expires; - - get_random_bytes(&entropy, sizeof(entropy)); - prandom_seed(entropy); - - /* reseed every ~60 seconds, in [40 .. 80) interval with slack */ - expires = 40 + prandom_u32_max(40); - seed_timer.expires = jiffies + msecs_to_jiffies(expires * MSEC_PER_SEC); - - add_timer(&seed_timer); -} - -static void __init __prandom_start_seed_timer(void) -{ - seed_timer.expires = jiffies + msecs_to_jiffies(40 * MSEC_PER_SEC); - add_timer(&seed_timer); -} - void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) { int i; @@ -257,51 +127,6 @@ void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) } EXPORT_SYMBOL(prandom_seed_full_state); -/* - * Generate better values after random number generator - * is fully initialized. - */ -static void __prandom_reseed(bool late) -{ - unsigned long flags; - static bool latch = false; - static DEFINE_SPINLOCK(lock); - - /* Asking for random bytes might result in bytes getting - * moved into the nonblocking pool and thus marking it - * as initialized. In this case we would double back into - * this function and attempt to do a late reseed. - * Ignore the pointless attempt to reseed again if we're - * already waiting for bytes when the nonblocking pool - * got initialized. - */ - - /* only allow initial seeding (late == false) once */ - if (!spin_trylock_irqsave(&lock, flags)) - return; - - if (latch && !late) - goto out; - - latch = true; - prandom_seed_full_state(&net_rand_state); -out: - spin_unlock_irqrestore(&lock, flags); -} - -void prandom_reseed_late(void) -{ - __prandom_reseed(true); -} - -static int __init prandom_reseed(void) -{ - __prandom_reseed(false); - __prandom_start_seed_timer(); - return 0; -} -late_initcall(prandom_reseed); - #ifdef CONFIG_RANDOM32_SELFTEST static struct prandom_test1 { u32 seed; @@ -421,7 +246,28 @@ static struct prandom_test2 { { 407983964U, 921U, 728767059U }, }; -static void __init prandom_state_selftest(void) +static u32 __extract_hwseed(void) +{ + unsigned int val = 0; + + (void)(arch_get_random_seed_int(&val) || + arch_get_random_int(&val)); + + return val; +} + +static void prandom_seed_early(struct rnd_state *state, u32 seed, + bool mix_with_hwseed) +{ +#define LCG(x) ((x) * 69069U) /* super-duper LCG */ +#define HWSEED() (mix_with_hwseed ? __extract_hwseed() : 0) + state->s1 = __seed(HWSEED() ^ LCG(seed), 2U); + state->s2 = __seed(HWSEED() ^ LCG(state->s1), 8U); + state->s3 = __seed(HWSEED() ^ LCG(state->s2), 16U); + state->s4 = __seed(HWSEED() ^ LCG(state->s3), 128U); +} + +static int __init prandom_state_selftest(void) { int i, j, errors = 0, runs = 0; bool error = false; @@ -461,5 +307,327 @@ static void __init prandom_state_selftest(void) pr_warn("prandom: %d/%d self tests failed\n", errors, runs); else pr_info("prandom: %d self tests passed\n", runs); + return 0; +} +core_initcall(prandom_state_selftest); +#endif + +/* + * The prandom_u32() implementation is now completely separate from the + * prandom_state() functions, which are retained (for now) for compatibility. + * + * Because of (ab)use in the networking code for choosing random TCP/UDP port + * numbers, which open DoS possibilities if guessable, we want something + * stronger than a standard PRNG. But the performance requirements of + * the network code do not allow robust crypto for this application. + * + * So this is a homebrew Junior Spaceman implementation, based on the + * lowest-latency trustworthy crypto primitive available, SipHash. + * (The authors of SipHash have not been consulted about this abuse of + * their work.) + * + * Standard SipHash-2-4 uses 2n+4 rounds to hash n words of input to + * one word of output. This abbreviated version uses 2 rounds per word + * of output. + */ + +struct siprand_state { + unsigned long v0; + unsigned long v1; + unsigned long v2; + unsigned long v3; +}; + +static DEFINE_PER_CPU(struct siprand_state, net_rand_state) __latent_entropy; +DEFINE_PER_CPU(unsigned long, net_rand_noise); +EXPORT_PER_CPU_SYMBOL(net_rand_noise); + +/* + * This is the core CPRNG function. As "pseudorandom", this is not used + * for truly valuable things, just intended to be a PITA to guess. + * For maximum speed, we do just two SipHash rounds per word. This is + * the same rate as 4 rounds per 64 bits that SipHash normally uses, + * so hopefully it's reasonably secure. + * + * There are two changes from the official SipHash finalization: + * - We omit some constants XORed with v2 in the SipHash spec as irrelevant; + * they are there only to make the output rounds distinct from the input + * rounds, and this application has no input rounds. + * - Rather than returning v0^v1^v2^v3, return v1+v3. + * If you look at the SipHash round, the last operation on v3 is + * "v3 ^= v0", so "v0 ^ v3" just undoes that, a waste of time. + * Likewise "v1 ^= v2". (The rotate of v2 makes a difference, but + * it still cancels out half of the bits in v2 for no benefit.) + * Second, since the last combining operation was xor, continue the + * pattern of alternating xor/add for a tiny bit of extra non-linearity. + */ +static inline u32 siprand_u32(struct siprand_state *s) +{ + unsigned long v0 = s->v0, v1 = s->v1, v2 = s->v2, v3 = s->v3; + unsigned long n = raw_cpu_read(net_rand_noise); + + v3 ^= n; + PRND_SIPROUND(v0, v1, v2, v3); + PRND_SIPROUND(v0, v1, v2, v3); + v0 ^= n; + s->v0 = v0; s->v1 = v1; s->v2 = v2; s->v3 = v3; + return v1 + v3; +} + + +/** + * prandom_u32 - pseudo random number generator + * + * A 32 bit pseudo-random number is generated using a fast + * algorithm suitable for simulation. This algorithm is NOT + * considered safe for cryptographic use. + */ +u32 prandom_u32(void) +{ + struct siprand_state *state = get_cpu_ptr(&net_rand_state); + u32 res = siprand_u32(state); + + trace_prandom_u32(res); + put_cpu_ptr(&net_rand_state); + return res; +} +EXPORT_SYMBOL(prandom_u32); + +/** + * prandom_bytes - get the requested number of pseudo-random bytes + * @buf: where to copy the pseudo-random bytes to + * @bytes: the requested number of bytes + */ +void prandom_bytes(void *buf, size_t bytes) +{ + struct siprand_state *state = get_cpu_ptr(&net_rand_state); + u8 *ptr = buf; + + while (bytes >= sizeof(u32)) { + put_unaligned(siprand_u32(state), (u32 *)ptr); + ptr += sizeof(u32); + bytes -= sizeof(u32); + } + + if (bytes > 0) { + u32 rem = siprand_u32(state); + + do { + *ptr++ = (u8)rem; + rem >>= BITS_PER_BYTE; + } while (--bytes > 0); + } + put_cpu_ptr(&net_rand_state); +} +EXPORT_SYMBOL(prandom_bytes); + +/** + * prandom_seed - add entropy to pseudo random number generator + * @entropy: entropy value + * + * Add some additional seed material to the prandom pool. + * The "entropy" is actually our IP address (the only caller is + * the network code), not for unpredictability, but to ensure that + * different machines are initialized differently. + */ +void prandom_seed(u32 entropy) +{ + int i; + + add_device_randomness(&entropy, sizeof(entropy)); + + for_each_possible_cpu(i) { + struct siprand_state *state = per_cpu_ptr(&net_rand_state, i); + unsigned long v0 = state->v0, v1 = state->v1; + unsigned long v2 = state->v2, v3 = state->v3; + + do { + v3 ^= entropy; + PRND_SIPROUND(v0, v1, v2, v3); + PRND_SIPROUND(v0, v1, v2, v3); + v0 ^= entropy; + } while (unlikely(!v0 || !v1 || !v2 || !v3)); + + WRITE_ONCE(state->v0, v0); + WRITE_ONCE(state->v1, v1); + WRITE_ONCE(state->v2, v2); + WRITE_ONCE(state->v3, v3); + } +} +EXPORT_SYMBOL(prandom_seed); + +/* + * Generate some initially weak seeding values to allow + * the prandom_u32() engine to be started. + */ +static int __init prandom_init_early(void) +{ + int i; + unsigned long v0, v1, v2, v3; + + if (!arch_get_random_long(&v0)) + v0 = jiffies; + if (!arch_get_random_long(&v1)) + v1 = random_get_entropy(); + v2 = v0 ^ PRND_K0; + v3 = v1 ^ PRND_K1; + + for_each_possible_cpu(i) { + struct siprand_state *state; + + v3 ^= i; + PRND_SIPROUND(v0, v1, v2, v3); + PRND_SIPROUND(v0, v1, v2, v3); + v0 ^= i; + + state = per_cpu_ptr(&net_rand_state, i); + state->v0 = v0; state->v1 = v1; + state->v2 = v2; state->v3 = v3; + } + + return 0; } +core_initcall(prandom_init_early); + + +/* Stronger reseeding when available, and periodically thereafter. */ +static void prandom_reseed(struct timer_list *unused); + +static DEFINE_TIMER(seed_timer, prandom_reseed); + +static void prandom_reseed(struct timer_list *unused) +{ + unsigned long expires; + int i; + + /* + * Reinitialize each CPU's PRNG with 128 bits of key. + * No locking on the CPUs, but then somewhat random results are, + * well, expected. + */ + for_each_possible_cpu(i) { + struct siprand_state *state; + unsigned long v0 = get_random_long(), v2 = v0 ^ PRND_K0; + unsigned long v1 = get_random_long(), v3 = v1 ^ PRND_K1; +#if BITS_PER_LONG == 32 + int j; + + /* + * On 32-bit machines, hash in two extra words to + * approximate 128-bit key length. Not that the hash + * has that much security, but this prevents a trivial + * 64-bit brute force. + */ + for (j = 0; j < 2; j++) { + unsigned long m = get_random_long(); + + v3 ^= m; + PRND_SIPROUND(v0, v1, v2, v3); + PRND_SIPROUND(v0, v1, v2, v3); + v0 ^= m; + } #endif + /* + * Probably impossible in practice, but there is a + * theoretical risk that a race between this reseeding + * and the target CPU writing its state back could + * create the all-zero SipHash fixed point. + * + * To ensure that never happens, ensure the state + * we write contains no zero words. + */ + state = per_cpu_ptr(&net_rand_state, i); + WRITE_ONCE(state->v0, v0 ? v0 : -1ul); + WRITE_ONCE(state->v1, v1 ? v1 : -1ul); + WRITE_ONCE(state->v2, v2 ? v2 : -1ul); + WRITE_ONCE(state->v3, v3 ? v3 : -1ul); + } + + /* reseed every ~60 seconds, in [40 .. 80) interval with slack */ + expires = round_jiffies(jiffies + 40 * HZ + prandom_u32_max(40 * HZ)); + mod_timer(&seed_timer, expires); +} + +/* + * The random ready callback can be called from almost any interrupt. + * To avoid worrying about whether it's safe to delay that interrupt + * long enough to seed all CPUs, just schedule an immediate timer event. + */ +static void prandom_timer_start(struct random_ready_callback *unused) +{ + mod_timer(&seed_timer, jiffies); +} + +#ifdef CONFIG_RANDOM32_SELFTEST +/* Principle: True 32-bit random numbers will all have 16 differing bits on + * average. For each 32-bit number, there are 601M numbers differing by 16 + * bits, and 89% of the numbers differ by at least 12 bits. Note that more + * than 16 differing bits also implies a correlation with inverted bits. Thus + * we take 1024 random numbers and compare each of them to the other ones, + * counting the deviation of correlated bits to 16. Constants report 32, + * counters 32-log2(TEST_SIZE), and pure randoms, around 6 or lower. With the + * u32 total, TEST_SIZE may be as large as 4096 samples. + */ +#define TEST_SIZE 1024 +static int __init prandom32_state_selftest(void) +{ + unsigned int x, y, bits, samples; + u32 xor, flip; + u32 total; + u32 *data; + + data = kmalloc(sizeof(*data) * TEST_SIZE, GFP_KERNEL); + if (!data) + return 0; + + for (samples = 0; samples < TEST_SIZE; samples++) + data[samples] = prandom_u32(); + + flip = total = 0; + for (x = 0; x < samples; x++) { + for (y = 0; y < samples; y++) { + if (x == y) + continue; + xor = data[x] ^ data[y]; + flip |= xor; + bits = hweight32(xor); + total += (bits - 16) * (bits - 16); + } + } + + /* We'll return the average deviation as 2*sqrt(corr/samples), which + * is also sqrt(4*corr/samples) which provides a better resolution. + */ + bits = int_sqrt(total / (samples * (samples - 1)) * 4); + if (bits > 6) + pr_warn("prandom32: self test failed (at least %u bits" + " correlated, fixed_mask=%#x fixed_value=%#x\n", + bits, ~flip, data[0] & ~flip); + else + pr_info("prandom32: self test passed (less than %u bits" + " correlated)\n", + bits+1); + kfree(data); + return 0; +} +core_initcall(prandom32_state_selftest); +#endif /* CONFIG_RANDOM32_SELFTEST */ + +/* + * Start periodic full reseeding as soon as strong + * random numbers are available. + */ +static int __init prandom_init_late(void) +{ + static struct random_ready_callback random_ready = { + .func = prandom_timer_start + }; + int ret = add_random_ready_callback(&random_ready); + + if (ret == -EALREADY) { + prandom_timer_start(&random_ready); + ret = 0; + } + return ret; +} +late_initcall(prandom_init_late); diff --git a/lib/rbtree.c b/lib/rbtree.c index abc86c6a3177..c4ac5c2421f2 100644 --- a/lib/rbtree.c +++ b/lib/rbtree.c @@ -13,7 +13,7 @@ #include <linux/export.h> /* - * red-black trees properties: http://en.wikipedia.org/wiki/Rbtree + * red-black trees properties: https://en.wikipedia.org/wiki/Rbtree * * 1) A node is either red or black * 2) The root is black @@ -503,7 +503,7 @@ struct rb_node *rb_next(const struct rb_node *node) if (node->rb_right) { node = node->rb_right; while (node->rb_left) - node=node->rb_left; + node = node->rb_left; return (struct rb_node *)node; } @@ -535,7 +535,7 @@ struct rb_node *rb_prev(const struct rb_node *node) if (node->rb_left) { node = node->rb_left; while (node->rb_right) - node=node->rb_right; + node = node->rb_right; return (struct rb_node *)node; } diff --git a/lib/reed_solomon/test_rslib.c b/lib/reed_solomon/test_rslib.c index 4eb29f365ece..d9d1c33aebda 100644 --- a/lib/reed_solomon/test_rslib.c +++ b/lib/reed_solomon/test_rslib.c @@ -385,7 +385,7 @@ static void test_bc(struct rs_control *rs, int len, int errs, /* * We check that the returned word is actually a - * codeword. The obious way to do this would be to + * codeword. The obvious way to do this would be to * compute the syndrome, but we don't want to replicate * that code here. However, all the codes are in * systematic form, and therefore we can encode the diff --git a/lib/refcount.c b/lib/refcount.c index ebac8b7d15a7..a207a8f22b3c 100644 --- a/lib/refcount.c +++ b/lib/refcount.c @@ -164,7 +164,7 @@ EXPORT_SYMBOL(refcount_dec_and_lock); * @flags: saved IRQ-flags if the is acquired * * Same as refcount_dec_and_lock() above except that the spinlock is acquired - * with disabled interupts. + * with disabled interrupts. * * Return: true and hold spinlock if able to decrement refcount to 0, false * otherwise diff --git a/lib/rhashtable.c b/lib/rhashtable.c index bdb7e4cadf05..e12bbfb240b8 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -31,7 +31,7 @@ union nested_table { union nested_table __rcu *table; - struct rhash_lock_head *bucket; + struct rhash_lock_head __rcu *bucket; }; static u32 head_hashfn(struct rhashtable *ht, @@ -63,13 +63,22 @@ EXPORT_SYMBOL_GPL(lockdep_rht_bucket_is_held); #define ASSERT_RHT_MUTEX(HT) #endif +static inline union nested_table *nested_table_top( + const struct bucket_table *tbl) +{ + /* The top-level bucket entry does not need RCU protection + * because it's set at the same time as tbl->nest. + */ + return (void *)rcu_dereference_protected(tbl->buckets[0], 1); +} + static void nested_table_free(union nested_table *ntbl, unsigned int size) { const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); const unsigned int len = 1 << shift; unsigned int i; - ntbl = rcu_dereference_raw(ntbl->table); + ntbl = rcu_dereference_protected(ntbl->table, 1); if (!ntbl) return; @@ -89,7 +98,7 @@ static void nested_bucket_table_free(const struct bucket_table *tbl) union nested_table *ntbl; unsigned int i; - ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]); + ntbl = nested_table_top(tbl); for (i = 0; i < len; i++) nested_table_free(ntbl + i, size); @@ -213,7 +222,7 @@ static struct bucket_table *rhashtable_last_table(struct rhashtable *ht, } static int rhashtable_rehash_one(struct rhashtable *ht, - struct rhash_lock_head **bkt, + struct rhash_lock_head __rcu **bkt, unsigned int old_hash) { struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); @@ -266,7 +275,7 @@ static int rhashtable_rehash_chain(struct rhashtable *ht, unsigned int old_hash) { struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); - struct rhash_lock_head **bkt = rht_bucket_var(old_tbl, old_hash); + struct rhash_lock_head __rcu **bkt = rht_bucket_var(old_tbl, old_hash); int err; if (!bkt) @@ -476,7 +485,7 @@ fail: } static void *rhashtable_lookup_one(struct rhashtable *ht, - struct rhash_lock_head **bkt, + struct rhash_lock_head __rcu **bkt, struct bucket_table *tbl, unsigned int hash, const void *key, struct rhash_head *obj) { @@ -526,12 +535,10 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, return ERR_PTR(-ENOENT); } -static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht, - struct rhash_lock_head **bkt, - struct bucket_table *tbl, - unsigned int hash, - struct rhash_head *obj, - void *data) +static struct bucket_table *rhashtable_insert_one( + struct rhashtable *ht, struct rhash_lock_head __rcu **bkt, + struct bucket_table *tbl, unsigned int hash, struct rhash_head *obj, + void *data) { struct bucket_table *new_tbl; struct rhash_head *head; @@ -582,7 +589,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, { struct bucket_table *new_tbl; struct bucket_table *tbl; - struct rhash_lock_head **bkt; + struct rhash_lock_head __rcu **bkt; unsigned int hash; void *data; @@ -696,7 +703,7 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_exit); * * Returns zero if successful. * - * Returns -EAGAIN if resize event occured. Note that the iterator + * Returns -EAGAIN if resize event occurred. Note that the iterator * will rewind back to the beginning and you may use it immediately * by calling rhashtable_walk_next. * @@ -1164,8 +1171,8 @@ void rhashtable_destroy(struct rhashtable *ht) } EXPORT_SYMBOL_GPL(rhashtable_destroy); -struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl, - unsigned int hash) +struct rhash_lock_head __rcu **__rht_bucket_nested( + const struct bucket_table *tbl, unsigned int hash) { const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); unsigned int index = hash & ((1 << tbl->nest) - 1); @@ -1173,7 +1180,7 @@ struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl, unsigned int subhash = hash; union nested_table *ntbl; - ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]); + ntbl = nested_table_top(tbl); ntbl = rht_dereference_bucket_rcu(ntbl[index].table, tbl, hash); subhash >>= tbl->nest; @@ -1193,10 +1200,10 @@ struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl, } EXPORT_SYMBOL_GPL(__rht_bucket_nested); -struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl, - unsigned int hash) +struct rhash_lock_head __rcu **rht_bucket_nested( + const struct bucket_table *tbl, unsigned int hash) { - static struct rhash_lock_head *rhnull; + static struct rhash_lock_head __rcu *rhnull; if (!rhnull) INIT_RHT_NULLS_HEAD(rhnull); @@ -1204,16 +1211,15 @@ struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl, } EXPORT_SYMBOL_GPL(rht_bucket_nested); -struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht, - struct bucket_table *tbl, - unsigned int hash) +struct rhash_lock_head __rcu **rht_bucket_nested_insert( + struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash) { const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); unsigned int index = hash & ((1 << tbl->nest) - 1); unsigned int size = tbl->size >> tbl->nest; union nested_table *ntbl; - ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]); + ntbl = nested_table_top(tbl); hash >>= tbl->nest; ntbl = nested_table_alloc(ht, &ntbl[index].table, size <= (1 << shift)); diff --git a/lib/sbitmap.c b/lib/sbitmap.c index af88d1346dd7..2709ab825499 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -9,57 +9,87 @@ #include <linux/sbitmap.h> #include <linux/seq_file.h> +static int init_alloc_hint(struct sbitmap *sb, gfp_t flags) +{ + unsigned depth = sb->depth; + + sb->alloc_hint = alloc_percpu_gfp(unsigned int, flags); + if (!sb->alloc_hint) + return -ENOMEM; + + if (depth && !sb->round_robin) { + int i; + + for_each_possible_cpu(i) + *per_cpu_ptr(sb->alloc_hint, i) = prandom_u32() % depth; + } + return 0; +} + +static inline unsigned update_alloc_hint_before_get(struct sbitmap *sb, + unsigned int depth) +{ + unsigned hint; + + hint = this_cpu_read(*sb->alloc_hint); + if (unlikely(hint >= depth)) { + hint = depth ? prandom_u32() % depth : 0; + this_cpu_write(*sb->alloc_hint, hint); + } + + return hint; +} + +static inline void update_alloc_hint_after_get(struct sbitmap *sb, + unsigned int depth, + unsigned int hint, + unsigned int nr) +{ + if (nr == -1) { + /* If the map is full, a hint won't do us much good. */ + this_cpu_write(*sb->alloc_hint, 0); + } else if (nr == hint || unlikely(sb->round_robin)) { + /* Only update the hint if we used it. */ + hint = nr + 1; + if (hint >= depth - 1) + hint = 0; + this_cpu_write(*sb->alloc_hint, hint); + } +} + /* * See if we have deferred clears that we can batch move */ -static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) +static inline bool sbitmap_deferred_clear(struct sbitmap_word *map) { - unsigned long mask, val; - bool ret = false; - unsigned long flags; - - spin_lock_irqsave(&sb->map[index].swap_lock, flags); + unsigned long mask; - if (!sb->map[index].cleared) - goto out_unlock; + if (!READ_ONCE(map->cleared)) + return false; /* * First get a stable cleared mask, setting the old mask to 0. */ - mask = xchg(&sb->map[index].cleared, 0); + mask = xchg(&map->cleared, 0); /* * Now clear the masked bits in our free word */ - do { - val = sb->map[index].word; - } while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val); - - ret = true; -out_unlock: - spin_unlock_irqrestore(&sb->map[index].swap_lock, flags); - return ret; + atomic_long_andnot(mask, (atomic_long_t *)&map->word); + BUILD_BUG_ON(sizeof(atomic_long_t) != sizeof(map->word)); + return true; } int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, - gfp_t flags, int node) + gfp_t flags, int node, bool round_robin, + bool alloc_hint) { unsigned int bits_per_word; unsigned int i; - if (shift < 0) { - shift = ilog2(BITS_PER_LONG); - /* - * If the bitmap is small, shrink the number of bits per word so - * we spread over a few cachelines, at least. If less than 4 - * bits, just forget about it, it's not going to work optimally - * anyway. - */ - if (depth >= 4) { - while ((4U << shift) > depth) - shift--; - } - } + if (shift < 0) + shift = sbitmap_calculate_shift(depth); + bits_per_word = 1U << shift; if (bits_per_word > BITS_PER_LONG) return -EINVAL; @@ -67,20 +97,29 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, sb->shift = shift; sb->depth = depth; sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word); + sb->round_robin = round_robin; if (depth == 0) { sb->map = NULL; return 0; } + if (alloc_hint) { + if (init_alloc_hint(sb, flags)) + return -ENOMEM; + } else { + sb->alloc_hint = NULL; + } + sb->map = kcalloc_node(sb->map_nr, sizeof(*sb->map), flags, node); - if (!sb->map) + if (!sb->map) { + free_percpu(sb->alloc_hint); return -ENOMEM; + } for (i = 0; i < sb->map_nr; i++) { sb->map[i].depth = min(depth, bits_per_word); depth -= sb->map[i].depth; - spin_lock_init(&sb->map[i].swap_lock); } return 0; } @@ -92,7 +131,7 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth) unsigned int i; for (i = 0; i < sb->map_nr; i++) - sbitmap_deferred_clear(sb, i); + sbitmap_deferred_clear(&sb->map[i]); sb->depth = depth; sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word); @@ -107,9 +146,11 @@ EXPORT_SYMBOL_GPL(sbitmap_resize); static int __sbitmap_get_word(unsigned long *word, unsigned long depth, unsigned int hint, bool wrap) { - unsigned int orig_hint = hint; int nr; + /* don't wrap if starting from 0 */ + wrap = wrap && hint; + while (1) { nr = find_next_zero_bit(word, depth, hint); if (unlikely(nr >= depth)) { @@ -118,8 +159,8 @@ static int __sbitmap_get_word(unsigned long *word, unsigned long depth, * offset to 0 in a failure case, so start from 0 to * exhaust the map. */ - if (orig_hint && hint && wrap) { - hint = orig_hint = 0; + if (hint && wrap) { + hint = 0; continue; } return -1; @@ -137,24 +178,24 @@ static int __sbitmap_get_word(unsigned long *word, unsigned long depth, } static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index, - unsigned int alloc_hint, bool round_robin) + unsigned int alloc_hint) { + struct sbitmap_word *map = &sb->map[index]; int nr; do { - nr = __sbitmap_get_word(&sb->map[index].word, - sb->map[index].depth, alloc_hint, - !round_robin); + nr = __sbitmap_get_word(&map->word, map->depth, alloc_hint, + !sb->round_robin); if (nr != -1) break; - if (!sbitmap_deferred_clear(sb, index)) + if (!sbitmap_deferred_clear(map)) break; } while (1); return nr; } -int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin) +static int __sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint) { unsigned int i, index; int nr = -1; @@ -166,14 +207,13 @@ int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin) * alloc_hint to find the right word index. No point in looping * twice in find_next_zero_bit() for that case. */ - if (round_robin) + if (sb->round_robin) alloc_hint = SB_NR_TO_BIT(sb, alloc_hint); else alloc_hint = 0; for (i = 0; i < sb->map_nr; i++) { - nr = sbitmap_find_bit_in_index(sb, index, alloc_hint, - round_robin); + nr = sbitmap_find_bit_in_index(sb, index, alloc_hint); if (nr != -1) { nr += index << sb->shift; break; @@ -187,10 +227,27 @@ int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin) return nr; } + +int sbitmap_get(struct sbitmap *sb) +{ + int nr; + unsigned int hint, depth; + + if (WARN_ON_ONCE(unlikely(!sb->alloc_hint))) + return -1; + + depth = READ_ONCE(sb->depth); + hint = update_alloc_hint_before_get(sb, depth); + nr = __sbitmap_get(sb, hint); + update_alloc_hint_after_get(sb, depth, hint, nr); + + return nr; +} EXPORT_SYMBOL_GPL(sbitmap_get); -int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint, - unsigned long shallow_depth) +static int __sbitmap_get_shallow(struct sbitmap *sb, + unsigned int alloc_hint, + unsigned long shallow_depth) { unsigned int i, index; int nr = -1; @@ -207,7 +264,7 @@ again: break; } - if (sbitmap_deferred_clear(sb, index)) + if (sbitmap_deferred_clear(&sb->map[index])) goto again; /* Jump to next index. */ @@ -222,6 +279,22 @@ again: return nr; } + +int sbitmap_get_shallow(struct sbitmap *sb, unsigned long shallow_depth) +{ + int nr; + unsigned int hint, depth; + + if (WARN_ON_ONCE(unlikely(!sb->alloc_hint))) + return -1; + + depth = READ_ONCE(sb->depth); + hint = update_alloc_hint_before_get(sb, depth); + nr = __sbitmap_get_shallow(sb, hint, shallow_depth); + update_alloc_hint_after_get(sb, depth, hint, nr); + + return nr; +} EXPORT_SYMBOL_GPL(sbitmap_get_shallow); bool sbitmap_any_bit_set(const struct sbitmap *sb) @@ -251,20 +324,21 @@ static unsigned int __sbitmap_weight(const struct sbitmap *sb, bool set) return weight; } -static unsigned int sbitmap_weight(const struct sbitmap *sb) +static unsigned int sbitmap_cleared(const struct sbitmap *sb) { - return __sbitmap_weight(sb, true); + return __sbitmap_weight(sb, false); } -static unsigned int sbitmap_cleared(const struct sbitmap *sb) +unsigned int sbitmap_weight(const struct sbitmap *sb) { - return __sbitmap_weight(sb, false); + return __sbitmap_weight(sb, true) - sbitmap_cleared(sb); } +EXPORT_SYMBOL_GPL(sbitmap_weight); void sbitmap_show(struct sbitmap *sb, struct seq_file *m) { seq_printf(m, "depth=%u\n", sb->depth); - seq_printf(m, "busy=%u\n", sbitmap_weight(sb) - sbitmap_cleared(sb)); + seq_printf(m, "busy=%u\n", sbitmap_weight(sb)); seq_printf(m, "cleared=%u\n", sbitmap_cleared(sb)); seq_printf(m, "bits_per_word=%u\n", 1U << sb->shift); seq_printf(m, "map_nr=%u\n", sb->map_nr); @@ -292,8 +366,11 @@ void sbitmap_bitmap_show(struct sbitmap *sb, struct seq_file *m) for (i = 0; i < sb->map_nr; i++) { unsigned long word = READ_ONCE(sb->map[i].word); + unsigned long cleared = READ_ONCE(sb->map[i].cleared); unsigned int word_bits = READ_ONCE(sb->map[i].depth); + word &= ~cleared; + while (word_bits > 0) { unsigned int bits = min(8 - byte_bits, word_bits); @@ -355,21 +432,11 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, int ret; int i; - ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node); + ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node, + round_robin, true); if (ret) return ret; - sbq->alloc_hint = alloc_percpu_gfp(unsigned int, flags); - if (!sbq->alloc_hint) { - sbitmap_free(&sbq->sb); - return -ENOMEM; - } - - if (depth && !round_robin) { - for_each_possible_cpu(i) - *per_cpu_ptr(sbq->alloc_hint, i) = prandom_u32() % depth; - } - sbq->min_shallow_depth = UINT_MAX; sbq->wake_batch = sbq_calc_wake_batch(sbq, depth); atomic_set(&sbq->wake_index, 0); @@ -377,7 +444,6 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, sbq->ws = kzalloc_node(SBQ_WAIT_QUEUES * sizeof(*sbq->ws), flags, node); if (!sbq->ws) { - free_percpu(sbq->alloc_hint); sbitmap_free(&sbq->sb); return -ENOMEM; } @@ -387,7 +453,6 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, atomic_set(&sbq->ws[i].wait_cnt, sbq->wake_batch); } - sbq->round_robin = round_robin; return 0; } EXPORT_SYMBOL_GPL(sbitmap_queue_init_node); @@ -420,60 +485,67 @@ EXPORT_SYMBOL_GPL(sbitmap_queue_resize); int __sbitmap_queue_get(struct sbitmap_queue *sbq) { + return sbitmap_get(&sbq->sb); +} +EXPORT_SYMBOL_GPL(__sbitmap_queue_get); + +unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, + unsigned int *offset) +{ + struct sbitmap *sb = &sbq->sb; unsigned int hint, depth; - int nr; + unsigned long index, nr; + int i; - hint = this_cpu_read(*sbq->alloc_hint); - depth = READ_ONCE(sbq->sb.depth); - if (unlikely(hint >= depth)) { - hint = depth ? prandom_u32() % depth : 0; - this_cpu_write(*sbq->alloc_hint, hint); - } - nr = sbitmap_get(&sbq->sb, hint, sbq->round_robin); + if (unlikely(sb->round_robin)) + return 0; - if (nr == -1) { - /* If the map is full, a hint won't do us much good. */ - this_cpu_write(*sbq->alloc_hint, 0); - } else if (nr == hint || unlikely(sbq->round_robin)) { - /* Only update the hint if we used it. */ - hint = nr + 1; - if (hint >= depth - 1) - hint = 0; - this_cpu_write(*sbq->alloc_hint, hint); + depth = READ_ONCE(sb->depth); + hint = update_alloc_hint_before_get(sb, depth); + + index = SB_NR_TO_INDEX(sb, hint); + + for (i = 0; i < sb->map_nr; i++) { + struct sbitmap_word *map = &sb->map[index]; + unsigned long get_mask; + + sbitmap_deferred_clear(map); + if (map->word == (1UL << (map->depth - 1)) - 1) + continue; + + nr = find_first_zero_bit(&map->word, map->depth); + if (nr + nr_tags <= map->depth) { + atomic_long_t *ptr = (atomic_long_t *) &map->word; + int map_tags = min_t(int, nr_tags, map->depth); + unsigned long val, ret; + + get_mask = ((1UL << map_tags) - 1) << nr; + do { + val = READ_ONCE(map->word); + ret = atomic_long_cmpxchg(ptr, val, get_mask | val); + } while (ret != val); + get_mask = (get_mask & ~ret) >> nr; + if (get_mask) { + *offset = nr + (index << sb->shift); + update_alloc_hint_after_get(sb, depth, hint, + *offset + map_tags - 1); + return get_mask; + } + } + /* Jump to next index. */ + if (++index >= sb->map_nr) + index = 0; } - return nr; + return 0; } -EXPORT_SYMBOL_GPL(__sbitmap_queue_get); int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq, unsigned int shallow_depth) { - unsigned int hint, depth; - int nr; - WARN_ON_ONCE(shallow_depth < sbq->min_shallow_depth); - hint = this_cpu_read(*sbq->alloc_hint); - depth = READ_ONCE(sbq->sb.depth); - if (unlikely(hint >= depth)) { - hint = depth ? prandom_u32() % depth : 0; - this_cpu_write(*sbq->alloc_hint, hint); - } - nr = sbitmap_get_shallow(&sbq->sb, hint, shallow_depth); - - if (nr == -1) { - /* If the map is full, a hint won't do us much good. */ - this_cpu_write(*sbq->alloc_hint, 0); - } else if (nr == hint || unlikely(sbq->round_robin)) { - /* Only update the hint if we used it. */ - hint = nr + 1; - if (hint >= depth - 1) - hint = 0; - this_cpu_write(*sbq->alloc_hint, hint); - } - - return nr; + return sbitmap_get_shallow(&sbq->sb, shallow_depth); } EXPORT_SYMBOL_GPL(__sbitmap_queue_get_shallow); @@ -556,13 +628,53 @@ void sbitmap_queue_wake_up(struct sbitmap_queue *sbq) } EXPORT_SYMBOL_GPL(sbitmap_queue_wake_up); +static inline void sbitmap_update_cpu_hint(struct sbitmap *sb, int cpu, int tag) +{ + if (likely(!sb->round_robin && tag < sb->depth)) + data_race(*per_cpu_ptr(sb->alloc_hint, cpu) = tag); +} + +void sbitmap_queue_clear_batch(struct sbitmap_queue *sbq, int offset, + int *tags, int nr_tags) +{ + struct sbitmap *sb = &sbq->sb; + unsigned long *addr = NULL; + unsigned long mask = 0; + int i; + + smp_mb__before_atomic(); + for (i = 0; i < nr_tags; i++) { + const int tag = tags[i] - offset; + unsigned long *this_addr; + + /* since we're clearing a batch, skip the deferred map */ + this_addr = &sb->map[SB_NR_TO_INDEX(sb, tag)].word; + if (!addr) { + addr = this_addr; + } else if (addr != this_addr) { + atomic_long_andnot(mask, (atomic_long_t *) addr); + mask = 0; + addr = this_addr; + } + mask |= (1UL << SB_NR_TO_BIT(sb, tag)); + } + + if (mask) + atomic_long_andnot(mask, (atomic_long_t *) addr); + + smp_mb__after_atomic(); + sbitmap_queue_wake_up(sbq); + sbitmap_update_cpu_hint(&sbq->sb, raw_smp_processor_id(), + tags[nr_tags - 1] - offset); +} + void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, unsigned int cpu) { /* * Once the clear bit is set, the bit may be allocated out. * - * Orders READ/WRITE on the asssociated instance(such as request + * Orders READ/WRITE on the associated instance(such as request * of blk_mq) by this bit for avoiding race with re-allocation, * and its pair is the memory barrier implied in __sbitmap_get_word. * @@ -580,9 +692,7 @@ void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, */ smp_mb__after_atomic(); sbitmap_queue_wake_up(sbq); - - if (likely(!sbq->round_robin && nr < sbq->sb.depth)) - *per_cpu_ptr(sbq->alloc_hint, cpu) = nr; + sbitmap_update_cpu_hint(&sbq->sb, cpu, nr); } EXPORT_SYMBOL_GPL(sbitmap_queue_clear); @@ -620,7 +730,7 @@ void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m) if (!first) seq_puts(m, ", "); first = false; - seq_printf(m, "%u", *per_cpu_ptr(sbq->alloc_hint, i)); + seq_printf(m, "%u", *per_cpu_ptr(sbq->sb.alloc_hint, i)); } seq_puts(m, "}\n"); @@ -638,7 +748,7 @@ void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m) } seq_puts(m, "}\n"); - seq_printf(m, "round_robin=%d\n", sbq->round_robin); + seq_printf(m, "round_robin=%d\n", sbq->sb.round_robin); seq_printf(m, "min_shallow_depth=%u\n", sbq->min_shallow_depth); } EXPORT_SYMBOL_GPL(sbitmap_queue_show); diff --git a/lib/scatterlist.c b/lib/scatterlist.c index c2cf2c311b7d..d5e82e4a57ad 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -38,7 +38,7 @@ EXPORT_SYMBOL(sg_next); * @sg: The scatterlist * * Description: - * Allows to know how many entries are in sg, taking into acount + * Allows to know how many entries are in sg, taking into account * chaining as well * **/ @@ -59,7 +59,7 @@ EXPORT_SYMBOL(sg_nents); * * Description: * Determines the number of entries in sg that are required to meet - * the supplied length, taking into acount chaining as well + * the supplied length, taking into account chaining as well * * Returns: * the number of sg entries needed, negative error on failure @@ -182,6 +182,7 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents) * @nents_first_chunk: Number of entries int the (preallocated) first * scatterlist chunk, 0 means no such preallocated first chunk * @free_fn: Free function + * @num_ents: Number of entries in the table * * Description: * Free an sg table previously allocated and setup with @@ -190,7 +191,8 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents) * **/ void __sg_free_table(struct sg_table *table, unsigned int max_ents, - unsigned int nents_first_chunk, sg_free_fn *free_fn) + unsigned int nents_first_chunk, sg_free_fn *free_fn, + unsigned int num_ents) { struct scatterlist *sgl, *next; unsigned curr_max_ents = nents_first_chunk ?: max_ents; @@ -199,8 +201,8 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, return; sgl = table->sgl; - while (table->orig_nents) { - unsigned int alloc_size = table->orig_nents; + while (num_ents) { + unsigned int alloc_size = num_ents; unsigned int sg_size; /* @@ -218,7 +220,7 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, next = NULL; } - table->orig_nents -= sg_size; + num_ents -= sg_size; if (nents_first_chunk) nents_first_chunk = 0; else @@ -232,13 +234,27 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, EXPORT_SYMBOL(__sg_free_table); /** + * sg_free_append_table - Free a previously allocated append sg table. + * @table: The mapped sg append table header + * + **/ +void sg_free_append_table(struct sg_append_table *table) +{ + __sg_free_table(&table->sgt, SG_MAX_SINGLE_ALLOC, false, sg_kfree, + table->total_nents); +} +EXPORT_SYMBOL(sg_free_append_table); + + +/** * sg_free_table - Free a previously allocated sg table * @table: The mapped sg table header * **/ void sg_free_table(struct sg_table *table) { - __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree); + __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree, + table->orig_nents); } EXPORT_SYMBOL(sg_free_table); @@ -311,7 +327,7 @@ int __sg_alloc_table(struct sg_table *table, unsigned int nents, if (prv) table->nents = ++table->orig_nents; - return -ENOMEM; + return -ENOMEM; } sg_init_table(sg, alloc_size); @@ -359,45 +375,113 @@ int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, NULL, 0, gfp_mask, sg_kmalloc); if (unlikely(ret)) - __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); - + sg_free_table(table); return ret; } EXPORT_SYMBOL(sg_alloc_table); +static struct scatterlist *get_next_sg(struct sg_append_table *table, + struct scatterlist *cur, + unsigned long needed_sges, + gfp_t gfp_mask) +{ + struct scatterlist *new_sg, *next_sg; + unsigned int alloc_size; + + if (cur) { + next_sg = sg_next(cur); + /* Check if last entry should be keeped for chainning */ + if (!sg_is_last(next_sg) || needed_sges == 1) + return next_sg; + } + + alloc_size = min_t(unsigned long, needed_sges, SG_MAX_SINGLE_ALLOC); + new_sg = sg_kmalloc(alloc_size, gfp_mask); + if (!new_sg) + return ERR_PTR(-ENOMEM); + sg_init_table(new_sg, alloc_size); + if (cur) { + table->total_nents += alloc_size - 1; + __sg_chain(next_sg, new_sg); + } else { + table->sgt.sgl = new_sg; + table->total_nents = alloc_size; + } + return new_sg; +} + /** - * __sg_alloc_table_from_pages - Allocate and initialize an sg table from - * an array of pages - * @sgt: The sg table header to use - * @pages: Pointer to an array of page pointers - * @n_pages: Number of pages in the pages array + * sg_alloc_append_table_from_pages - Allocate and initialize an append sg + * table from an array of pages + * @sgt_append: The sg append table to use + * @pages: Pointer to an array of page pointers + * @n_pages: Number of pages in the pages array * @offset: Offset from start of the first page to the start of a buffer * @size: Number of valid bytes in the buffer (after offset) - * @max_segment: Maximum size of a scatterlist node in bytes (page aligned) + * @max_segment: Maximum size of a scatterlist element in bytes + * @left_pages: Left pages caller have to set after this call * @gfp_mask: GFP allocation mask * - * Description: - * Allocate and initialize an sg table from a list of pages. Contiguous - * ranges of the pages are squashed into a single scatterlist node up to the - * maximum size specified in @max_segment. An user may provide an offset at a - * start and a size of valid data in a buffer specified by the page array. - * The returned sg table is released by sg_free_table. + * Description: + * In the first call it allocate and initialize an sg table from a list of + * pages, else reuse the scatterlist from sgt_append. Contiguous ranges of + * the pages are squashed into a single scatterlist entry up to the maximum + * size specified in @max_segment. A user may provide an offset at a start + * and a size of valid data in a buffer specified by the page array. The + * returned sg table is released by sg_free_append_table * * Returns: * 0 on success, negative error on failure + * + * Notes: + * If this function returns non-0 (eg failure), the caller must call + * sg_free_append_table() to cleanup any leftover allocations. + * + * In the fist call, sgt_append must by initialized. */ -int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, - unsigned int n_pages, unsigned int offset, - unsigned long size, unsigned int max_segment, - gfp_t gfp_mask) +int sg_alloc_append_table_from_pages(struct sg_append_table *sgt_append, + struct page **pages, unsigned int n_pages, unsigned int offset, + unsigned long size, unsigned int max_segment, + unsigned int left_pages, gfp_t gfp_mask) { - unsigned int chunks, cur_page, seg_len, i; - int ret; - struct scatterlist *s; + unsigned int chunks, cur_page, seg_len, i, prv_len = 0; + unsigned int added_nents = 0; + struct scatterlist *s = sgt_append->prv; - if (WARN_ON(!max_segment || offset_in_page(max_segment))) + /* + * The algorithm below requires max_segment to be aligned to PAGE_SIZE + * otherwise it can overshoot. + */ + max_segment = ALIGN_DOWN(max_segment, PAGE_SIZE); + if (WARN_ON(max_segment < PAGE_SIZE)) return -EINVAL; + if (IS_ENABLED(CONFIG_ARCH_NO_SG_CHAIN) && sgt_append->prv) + return -EOPNOTSUPP; + + if (sgt_append->prv) { + unsigned long paddr = + (page_to_pfn(sg_page(sgt_append->prv)) * PAGE_SIZE + + sgt_append->prv->offset + sgt_append->prv->length) / + PAGE_SIZE; + + if (WARN_ON(offset)) + return -EINVAL; + + /* Merge contiguous pages into the last SG */ + prv_len = sgt_append->prv->length; + while (n_pages && page_to_pfn(pages[0]) == paddr) { + if (sgt_append->prv->length + PAGE_SIZE > max_segment) + break; + sgt_append->prv->length += PAGE_SIZE; + paddr++; + pages++; + n_pages--; + } + if (!n_pages) + goto out; + } + /* compute number of contiguous chunks */ chunks = 1; seg_len = 0; @@ -410,13 +494,9 @@ int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, } } - ret = sg_alloc_table(sgt, chunks, gfp_mask); - if (unlikely(ret)) - return ret; - /* merging chunks and putting them into the scatterlist */ cur_page = 0; - for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { + for (i = 0; i < chunks; i++) { unsigned int j, chunk_size; /* look for the end of the current chunk */ @@ -429,46 +509,78 @@ int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, break; } + /* Pass how many chunks might be left */ + s = get_next_sg(sgt_append, s, chunks - i + left_pages, + gfp_mask); + if (IS_ERR(s)) { + /* + * Adjust entry length to be as before function was + * called. + */ + if (sgt_append->prv) + sgt_append->prv->length = prv_len; + return PTR_ERR(s); + } chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset; sg_set_page(s, pages[cur_page], min_t(unsigned long, size, chunk_size), offset); + added_nents++; size -= chunk_size; offset = 0; cur_page = j; } - + sgt_append->sgt.nents += added_nents; + sgt_append->sgt.orig_nents = sgt_append->sgt.nents; + sgt_append->prv = s; +out: + if (!left_pages) + sg_mark_end(s); return 0; } -EXPORT_SYMBOL(__sg_alloc_table_from_pages); +EXPORT_SYMBOL(sg_alloc_append_table_from_pages); /** - * sg_alloc_table_from_pages - Allocate and initialize an sg table from - * an array of pages + * sg_alloc_table_from_pages_segment - Allocate and initialize an sg table from + * an array of pages and given maximum + * segment. * @sgt: The sg table header to use * @pages: Pointer to an array of page pointers * @n_pages: Number of pages in the pages array * @offset: Offset from start of the first page to the start of a buffer * @size: Number of valid bytes in the buffer (after offset) + * @max_segment: Maximum size of a scatterlist element in bytes * @gfp_mask: GFP allocation mask * * Description: * Allocate and initialize an sg table from a list of pages. Contiguous - * ranges of the pages are squashed into a single scatterlist node. A user - * may provide an offset at a start and a size of valid data in a buffer - * specified by the page array. The returned sg table is released by - * sg_free_table. + * ranges of the pages are squashed into a single scatterlist node up to the + * maximum size specified in @max_segment. A user may provide an offset at a + * start and a size of valid data in a buffer specified by the page array. * - * Returns: + * The returned sg table is released by sg_free_table. + * + * Returns: * 0 on success, negative error on failure */ -int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, - unsigned int n_pages, unsigned int offset, - unsigned long size, gfp_t gfp_mask) +int sg_alloc_table_from_pages_segment(struct sg_table *sgt, struct page **pages, + unsigned int n_pages, unsigned int offset, + unsigned long size, unsigned int max_segment, + gfp_t gfp_mask) { - return __sg_alloc_table_from_pages(sgt, pages, n_pages, offset, size, - SCATTERLIST_MAX_SEGMENT, gfp_mask); + struct sg_append_table append = {}; + int err; + + err = sg_alloc_append_table_from_pages(&append, pages, n_pages, offset, + size, max_segment, 0, gfp_mask); + if (err) { + sg_free_append_table(&append); + return err; + } + memcpy(sgt, &append.sgt, sizeof(*sgt)); + WARN_ON(append.total_nents != sgt->orig_nents); + return 0; } -EXPORT_SYMBOL(sg_alloc_table_from_pages); +EXPORT_SYMBOL(sg_alloc_table_from_pages_segment); #ifdef CONFIG_SGL_ALLOC @@ -504,7 +616,7 @@ struct scatterlist *sgl_alloc_order(unsigned long long length, nalloc++; } sgl = kmalloc_array(nalloc, sizeof(struct scatterlist), - (gfp & ~GFP_DMA) | __GFP_ZERO); + gfp & ~GFP_DMA); if (!sgl) return NULL; @@ -514,7 +626,7 @@ struct scatterlist *sgl_alloc_order(unsigned long long length, elem_len = min_t(u64, length, PAGE_SIZE << order); page = alloc_pages(gfp, order); if (!page) { - sgl_free(sgl); + sgl_free_order(sgl, order); return NULL; } @@ -716,8 +828,7 @@ static bool sg_miter_get_next_page(struct sg_mapping_iter *miter) * stops @miter. * * Context: - * Don't care if @miter is stopped, or not proceeded yet. - * Otherwise, preemption disabled if the SG_MITER_ATOMIC is set. + * Don't care. * * Returns: * true if @miter contains the valid mapping. false if end of sg @@ -753,8 +864,7 @@ EXPORT_SYMBOL(sg_miter_skip); * @miter->addr and @miter->length point to the current mapping. * * Context: - * Preemption disabled if SG_MITER_ATOMIC. Preemption must stay disabled - * till @miter is stopped. May sleep if !SG_MITER_ATOMIC. + * May sleep if !SG_MITER_ATOMIC. * * Returns: * true if @miter contains the next mapping. false if end of sg @@ -794,8 +904,7 @@ EXPORT_SYMBOL(sg_miter_next); * need to be released during iteration. * * Context: - * Preemption disabled if the SG_MITER_ATOMIC is set. Don't care - * otherwise. + * Don't care otherwise. */ void sg_miter_stop(struct sg_mapping_iter *miter) { @@ -806,12 +915,11 @@ void sg_miter_stop(struct sg_mapping_iter *miter) miter->__offset += miter->consumed; miter->__remaining -= miter->consumed; - if ((miter->__flags & SG_MITER_TO_SG) && - !PageSlab(miter->page)) - flush_kernel_dcache_page(miter->page); + if (miter->__flags & SG_MITER_TO_SG) + flush_dcache_page(miter->page); if (miter->__flags & SG_MITER_ATOMIC) { - WARN_ON_ONCE(preemptible()); + WARN_ON_ONCE(!pagefault_disabled()); kunmap_atomic(miter->addr); } else kunmap(miter->page); @@ -832,7 +940,7 @@ EXPORT_SYMBOL(sg_miter_stop); * @buflen: The number of bytes to copy * @skip: Number of bytes to skip before copying * @to_buffer: transfer direction (true == from an sg list to a - * buffer, false == from a buffer to an sg list + * buffer, false == from a buffer to an sg list) * * Returns the number of copied bytes. * @@ -852,7 +960,7 @@ size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf, sg_miter_start(&miter, sgl, nents, sg_flags); if (!sg_miter_skip(&miter, skip)) - return false; + return 0; while ((offset < buflen) && sg_miter_next(&miter)) { unsigned int len; diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 4e865d42ab03..0a68f7aa85d6 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -91,6 +91,7 @@ int seq_buf_printf(struct seq_buf *s, const char *fmt, ...) return ret; } +EXPORT_SYMBOL_GPL(seq_buf_printf); #ifdef CONFIG_BINARY_PRINTF /** @@ -228,8 +229,10 @@ int seq_buf_putmem_hex(struct seq_buf *s, const void *mem, WARN_ON(s->size == 0); + BUILD_BUG_ON(MAX_MEMHEX_BYTES * 2 >= HEX_CHARS); + while (len) { - start_len = min(len, HEX_CHARS - 1); + start_len = min(len, MAX_MEMHEX_BYTES); #ifdef __BIG_ENDIAN for (i = 0, j = 0; i < start_len; i++) { #else @@ -242,12 +245,14 @@ int seq_buf_putmem_hex(struct seq_buf *s, const void *mem, break; /* j increments twice per loop */ - len -= j / 2; hex[j++] = ' '; seq_buf_putmem(s, hex, j); if (seq_buf_has_overflowed(s)) return -1; + + len -= start_len; + data += start_len; } return 0; } @@ -284,7 +289,7 @@ int seq_buf_path(struct seq_buf *s, const struct path *path, const char *esc) } /** - * seq_buf_to_user - copy the squence buffer to user space + * seq_buf_to_user - copy the sequence buffer to user space * @s: seq_buf descriptor * @ubuf: The userspace memory location to copy to * @cnt: The amount to copy diff --git a/lib/sg_pool.c b/lib/sg_pool.c index db29e5c1f790..a0b1a52cd6f7 100644 --- a/lib/sg_pool.c +++ b/lib/sg_pool.c @@ -90,7 +90,8 @@ void sg_free_table_chained(struct sg_table *table, if (nents_first_chunk == 1) nents_first_chunk = 0; - __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free); + __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free, + table->orig_nents); } EXPORT_SYMBOL_GPL(sg_free_table_chained); diff --git a/lib/sha1.c b/lib/sha1.c index 1d96d2c02b82..9bd1935a1472 100644 --- a/lib/sha1.c +++ b/lib/sha1.c @@ -9,7 +9,7 @@ #include <linux/kernel.h> #include <linux/export.h> #include <linux/bitops.h> -#include <linux/cryptohash.h> +#include <crypto/sha1.h> #include <asm/unaligned.h> /* @@ -64,22 +64,24 @@ #define T_60_79(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) , 0xca62c1d6, A, B, C, D, E ) /** - * sha_transform - single block SHA1 transform + * sha1_transform - single block SHA1 transform (deprecated) * * @digest: 160 bit digest to update * @data: 512 bits of data to hash * @array: 16 words of workspace (see note) * - * This function generates a SHA1 digest for a single 512-bit block. - * Be warned, it does not handle padding and message digest, do not - * confuse it with the full FIPS 180-1 digest algorithm for variable - * length messages. + * This function executes SHA-1's internal compression function. It updates the + * 160-bit internal state (@digest) with a single 512-bit data block (@data). + * + * Don't use this function. SHA-1 is no longer considered secure. And even if + * you do have to use SHA-1, this isn't the correct way to hash something with + * SHA-1 as this doesn't handle padding and finalization. * * Note: If the hash is security sensitive, the caller should be sure * to clear the workspace. This is left to the caller to avoid * unnecessary clears between chained hashing operations. */ -void sha_transform(__u32 *digest, const char *data, __u32 *array) +void sha1_transform(__u32 *digest, const char *data, __u32 *array) { __u32 A, B, C, D, E; @@ -185,13 +187,13 @@ void sha_transform(__u32 *digest, const char *data, __u32 *array) digest[3] += D; digest[4] += E; } -EXPORT_SYMBOL(sha_transform); +EXPORT_SYMBOL(sha1_transform); /** - * sha_init - initialize the vectors for a SHA1 digest + * sha1_init - initialize the vectors for a SHA1 digest * @buf: vector to initialize */ -void sha_init(__u32 *buf) +void sha1_init(__u32 *buf) { buf[0] = 0x67452301; buf[1] = 0xefcdab89; @@ -199,4 +201,4 @@ void sha_init(__u32 *buf) buf[3] = 0x10325476; buf[4] = 0xc3d2e1f0; } -EXPORT_SYMBOL(sha_init); +EXPORT_SYMBOL(sha1_init); diff --git a/lib/siphash.c b/lib/siphash.c index c47bb6ff2149..a90112ee72a1 100644 --- a/lib/siphash.c +++ b/lib/siphash.c @@ -68,11 +68,11 @@ u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key) bytemask_from_count(left))); #else switch (left) { - case 7: b |= ((u64)end[6]) << 48; /* fall through */ - case 6: b |= ((u64)end[5]) << 40; /* fall through */ - case 5: b |= ((u64)end[4]) << 32; /* fall through */ + case 7: b |= ((u64)end[6]) << 48; fallthrough; + case 6: b |= ((u64)end[5]) << 40; fallthrough; + case 5: b |= ((u64)end[4]) << 32; fallthrough; case 4: b |= le32_to_cpup(data); break; - case 3: b |= ((u64)end[2]) << 16; /* fall through */ + case 3: b |= ((u64)end[2]) << 16; fallthrough; case 2: b |= le16_to_cpup(data); break; case 1: b |= end[0]; } @@ -101,11 +101,11 @@ u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key) bytemask_from_count(left))); #else switch (left) { - case 7: b |= ((u64)end[6]) << 48; /* fall through */ - case 6: b |= ((u64)end[5]) << 40; /* fall through */ - case 5: b |= ((u64)end[4]) << 32; /* fall through */ + case 7: b |= ((u64)end[6]) << 48; fallthrough; + case 6: b |= ((u64)end[5]) << 40; fallthrough; + case 5: b |= ((u64)end[4]) << 32; fallthrough; case 4: b |= get_unaligned_le32(end); break; - case 3: b |= ((u64)end[2]) << 16; /* fall through */ + case 3: b |= ((u64)end[2]) << 16; fallthrough; case 2: b |= get_unaligned_le16(end); break; case 1: b |= end[0]; } @@ -268,11 +268,11 @@ u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key) bytemask_from_count(left))); #else switch (left) { - case 7: b |= ((u64)end[6]) << 48; /* fall through */ - case 6: b |= ((u64)end[5]) << 40; /* fall through */ - case 5: b |= ((u64)end[4]) << 32; /* fall through */ + case 7: b |= ((u64)end[6]) << 48; fallthrough; + case 6: b |= ((u64)end[5]) << 40; fallthrough; + case 5: b |= ((u64)end[4]) << 32; fallthrough; case 4: b |= le32_to_cpup(data); break; - case 3: b |= ((u64)end[2]) << 16; /* fall through */ + case 3: b |= ((u64)end[2]) << 16; fallthrough; case 2: b |= le16_to_cpup(data); break; case 1: b |= end[0]; } @@ -301,11 +301,11 @@ u32 __hsiphash_unaligned(const void *data, size_t len, bytemask_from_count(left))); #else switch (left) { - case 7: b |= ((u64)end[6]) << 48; /* fall through */ - case 6: b |= ((u64)end[5]) << 40; /* fall through */ - case 5: b |= ((u64)end[4]) << 32; /* fall through */ + case 7: b |= ((u64)end[6]) << 48; fallthrough; + case 6: b |= ((u64)end[5]) << 40; fallthrough; + case 5: b |= ((u64)end[4]) << 32; fallthrough; case 4: b |= get_unaligned_le32(end); break; - case 3: b |= ((u64)end[2]) << 16; /* fall through */ + case 3: b |= ((u64)end[2]) << 16; fallthrough; case 2: b |= get_unaligned_le16(end); break; case 1: b |= end[0]; } @@ -431,7 +431,7 @@ u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key) v0 ^= m; } switch (left) { - case 3: b |= ((u32)end[2]) << 16; /* fall through */ + case 3: b |= ((u32)end[2]) << 16; fallthrough; case 2: b |= le16_to_cpup(data); break; case 1: b |= end[0]; } @@ -454,7 +454,7 @@ u32 __hsiphash_unaligned(const void *data, size_t len, v0 ^= m; } switch (left) { - case 3: b |= ((u32)end[2]) << 16; /* fall through */ + case 3: b |= ((u32)end[2]) << 16; fallthrough; case 2: b |= get_unaligned_le16(end); break; case 1: b |= end[0]; } diff --git a/lib/slub_kunit.c b/lib/slub_kunit.c new file mode 100644 index 000000000000..8662dc6cb509 --- /dev/null +++ b/lib/slub_kunit.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <kunit/test.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/kernel.h> +#include "../mm/slab.h" + +static struct kunit_resource resource; +static int slab_errors; + +static void test_clobber_zone(struct kunit *test) +{ + struct kmem_cache *s = kmem_cache_create("TestSlub_RZ_alloc", 64, 0, + SLAB_RED_ZONE, NULL); + u8 *p = kmem_cache_alloc(s, GFP_KERNEL); + + kasan_disable_current(); + p[64] = 0x12; + + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 2, slab_errors); + + kasan_enable_current(); + kmem_cache_free(s, p); + kmem_cache_destroy(s); +} + +#ifndef CONFIG_KASAN +static void test_next_pointer(struct kunit *test) +{ + struct kmem_cache *s = kmem_cache_create("TestSlub_next_ptr_free", 64, 0, + SLAB_POISON, NULL); + u8 *p = kmem_cache_alloc(s, GFP_KERNEL); + unsigned long tmp; + unsigned long *ptr_addr; + + kmem_cache_free(s, p); + + ptr_addr = (unsigned long *)(p + s->offset); + tmp = *ptr_addr; + p[s->offset] = 0x12; + + /* + * Expecting three errors. + * One for the corrupted freechain and the other one for the wrong + * count of objects in use. The third error is fixing broken cache. + */ + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 3, slab_errors); + + /* + * Try to repair corrupted freepointer. + * Still expecting two errors. The first for the wrong count + * of objects in use. + * The second error is for fixing broken cache. + */ + *ptr_addr = tmp; + slab_errors = 0; + + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 2, slab_errors); + + /* + * Previous validation repaired the count of objects in use. + * Now expecting no error. + */ + slab_errors = 0; + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 0, slab_errors); + + kmem_cache_destroy(s); +} + +static void test_first_word(struct kunit *test) +{ + struct kmem_cache *s = kmem_cache_create("TestSlub_1th_word_free", 64, 0, + SLAB_POISON, NULL); + u8 *p = kmem_cache_alloc(s, GFP_KERNEL); + + kmem_cache_free(s, p); + *p = 0x78; + + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 2, slab_errors); + + kmem_cache_destroy(s); +} + +static void test_clobber_50th_byte(struct kunit *test) +{ + struct kmem_cache *s = kmem_cache_create("TestSlub_50th_word_free", 64, 0, + SLAB_POISON, NULL); + u8 *p = kmem_cache_alloc(s, GFP_KERNEL); + + kmem_cache_free(s, p); + p[50] = 0x9a; + + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 2, slab_errors); + + kmem_cache_destroy(s); +} +#endif + +static void test_clobber_redzone_free(struct kunit *test) +{ + struct kmem_cache *s = kmem_cache_create("TestSlub_RZ_free", 64, 0, + SLAB_RED_ZONE, NULL); + u8 *p = kmem_cache_alloc(s, GFP_KERNEL); + + kasan_disable_current(); + kmem_cache_free(s, p); + p[64] = 0xab; + + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 2, slab_errors); + + kasan_enable_current(); + kmem_cache_destroy(s); +} + +static int test_init(struct kunit *test) +{ + slab_errors = 0; + + kunit_add_named_resource(test, NULL, NULL, &resource, + "slab_errors", &slab_errors); + return 0; +} + +static struct kunit_case test_cases[] = { + KUNIT_CASE(test_clobber_zone), + +#ifndef CONFIG_KASAN + KUNIT_CASE(test_next_pointer), + KUNIT_CASE(test_first_word), + KUNIT_CASE(test_clobber_50th_byte), +#endif + + KUNIT_CASE(test_clobber_redzone_free), + {} +}; + +static struct kunit_suite test_suite = { + .name = "slub_test", + .init = test_init, + .test_cases = test_cases, +}; +kunit_test_suite(test_suite); + +MODULE_LICENSE("GPL"); diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c index bd9571653288..046ac6297c78 100644 --- a/lib/smp_processor_id.c +++ b/lib/smp_processor_id.c @@ -8,7 +8,7 @@ #include <linux/kprobes.h> #include <linux/sched.h> -notrace static nokprobe_inline +noinstr static unsigned int check_preemption_disabled(const char *what1, const char *what2) { int this_cpu = raw_smp_processor_id(); @@ -19,12 +19,13 @@ unsigned int check_preemption_disabled(const char *what1, const char *what2) if (irqs_disabled()) goto out; - /* - * Kernel threads bound to a single CPU can safely use - * smp_processor_id(): - */ - if (current->nr_cpus_allowed == 1) + if (is_percpu_thread()) + goto out; + +#ifdef CONFIG_SMP + if (current->migration_disabled) goto out; +#endif /* * It is valid to assume CPU-locality during early bootup: @@ -37,6 +38,7 @@ unsigned int check_preemption_disabled(const char *what1, const char *what2) */ preempt_disable_notrace(); + instrumentation_begin(); if (!printk_ratelimit()) goto out_enable; @@ -45,6 +47,7 @@ unsigned int check_preemption_disabled(const char *what1, const char *what2) printk("caller is %pS\n", __builtin_return_address(0)); dump_stack(); + instrumentation_end(); out_enable: preempt_enable_no_resched_notrace(); @@ -52,16 +55,14 @@ out: return this_cpu; } -notrace unsigned int debug_smp_processor_id(void) +noinstr unsigned int debug_smp_processor_id(void) { return check_preemption_disabled("smp_processor_id", ""); } EXPORT_SYMBOL(debug_smp_processor_id); -NOKPROBE_SYMBOL(debug_smp_processor_id); -notrace void __this_cpu_preempt_check(const char *op) +noinstr void __this_cpu_preempt_check(const char *op) { check_preemption_disabled("__this_cpu_", op); } EXPORT_SYMBOL(__this_cpu_preempt_check); -NOKPROBE_SYMBOL(__this_cpu_preempt_check); diff --git a/lib/sort.c b/lib/sort.c index 3ad454411997..aa18153864d2 100644 --- a/lib/sort.c +++ b/lib/sort.c @@ -51,7 +51,7 @@ static bool is_aligned(const void *base, size_t size, unsigned char align) * which basically all CPUs have, to minimize loop overhead computations. * * For some reason, on x86 gcc 7.3.0 adds a redundant test of n at the - * bottom of the loop, even though the zero flag is stil valid from the + * bottom of the loop, even though the zero flag is still valid from the * subtract (since the intervening mov instructions don't alter the flags). * Gcc 8.1.0 doesn't have that problem. */ diff --git a/lib/stackdepot.c b/lib/stackdepot.c index ed717dd08ff3..b437ae79aca1 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -11,7 +11,7 @@ * Instead, stack depot maintains a hashtable of unique stacktraces. Since alloc * and free stacks repeat a lot, we save about 100x space. * Stacks are never removed from depot, so we store them contiguously one after - * another in a contiguos memory allocation. + * another in a contiguous memory allocation. * * Author: Alexander Potapenko <glider@google.com> * Copyright (C) 2016 Google, Inc. @@ -30,6 +30,7 @@ #include <linux/stackdepot.h> #include <linux/string.h> #include <linux/types.h> +#include <linux/memblock.h> #define DEPOT_STACK_BITS (sizeof(depot_stack_handle_t) * 8) @@ -61,7 +62,7 @@ struct stack_record { u32 hash; /* Hash in the hastable */ u32 size; /* Number of frames in the stack */ union handle_parts handle; - unsigned long entries[1]; /* Variable-sized array of entries. */ + unsigned long entries[]; /* Variable-sized array of entries. */ }; static void *stack_slabs[STACK_ALLOC_MAX_SLABS]; @@ -69,7 +70,7 @@ static void *stack_slabs[STACK_ALLOC_MAX_SLABS]; static int depot_index; static int next_slab_inited; static size_t depot_offset; -static DEFINE_SPINLOCK(depot_lock); +static DEFINE_RAW_SPINLOCK(depot_lock); static bool init_stack_slab(void **prealloc) { @@ -83,25 +84,28 @@ static bool init_stack_slab(void **prealloc) return true; if (stack_slabs[depot_index] == NULL) { stack_slabs[depot_index] = *prealloc; + *prealloc = NULL; } else { - stack_slabs[depot_index + 1] = *prealloc; + /* If this is the last depot slab, do not touch the next one. */ + if (depot_index + 1 < STACK_ALLOC_MAX_SLABS) { + stack_slabs[depot_index + 1] = *prealloc; + *prealloc = NULL; + } /* * This smp_store_release pairs with smp_load_acquire() from * |next_slab_inited| above and in stack_depot_save(). */ smp_store_release(&next_slab_inited, 1); } - *prealloc = NULL; return true; } /* Allocation of a new stack in raw storage */ -static struct stack_record *depot_alloc_stack(unsigned long *entries, int size, - u32 hash, void **prealloc, gfp_t alloc_flags) +static struct stack_record * +depot_alloc_stack(unsigned long *entries, int size, u32 hash, void **prealloc) { - int required_size = offsetof(struct stack_record, entries) + - sizeof(unsigned long) * size; struct stack_record *stack; + size_t required_size = struct_size(stack, entries, size); required_size = ALIGN(required_size, 1 << STACK_ALLOC_ALIGN); @@ -131,27 +135,51 @@ static struct stack_record *depot_alloc_stack(unsigned long *entries, int size, stack->handle.slabindex = depot_index; stack->handle.offset = depot_offset >> STACK_ALLOC_ALIGN; stack->handle.valid = 1; - memcpy(stack->entries, entries, size * sizeof(unsigned long)); + memcpy(stack->entries, entries, flex_array_size(stack, entries, size)); depot_offset += required_size; return stack; } -#define STACK_HASH_ORDER 20 -#define STACK_HASH_SIZE (1L << STACK_HASH_ORDER) +#define STACK_HASH_SIZE (1L << CONFIG_STACK_HASH_ORDER) #define STACK_HASH_MASK (STACK_HASH_SIZE - 1) #define STACK_HASH_SEED 0x9747b28c -static struct stack_record *stack_table[STACK_HASH_SIZE] = { - [0 ... STACK_HASH_SIZE - 1] = NULL -}; +static bool stack_depot_disable; +static struct stack_record **stack_table; + +static int __init is_stack_depot_disabled(char *str) +{ + int ret; + + ret = kstrtobool(str, &stack_depot_disable); + if (!ret && stack_depot_disable) { + pr_info("Stack Depot is disabled\n"); + stack_table = NULL; + } + return 0; +} +early_param("stack_depot_disable", is_stack_depot_disabled); + +int __init stack_depot_init(void) +{ + if (!stack_depot_disable) { + size_t size = (STACK_HASH_SIZE * sizeof(struct stack_record *)); + int i; + + stack_table = memblock_alloc(size, size); + for (i = 0; i < STACK_HASH_SIZE; i++) + stack_table[i] = NULL; + } + return 0; +} /* Calculate hash for a stack */ static inline u32 hash_stack(unsigned long *entries, unsigned int size) { return jhash2((u32 *)entries, - size * sizeof(unsigned long) / sizeof(u32), - STACK_HASH_SEED); + array_size(size, sizeof(*entries)) / sizeof(u32), + STACK_HASH_SEED); } /* Use our own, non-instrumented version of memcmp(). @@ -186,6 +214,49 @@ static inline struct stack_record *find_stack(struct stack_record *bucket, } /** + * stack_depot_snprint - print stack entries from a depot into a buffer + * + * @handle: Stack depot handle which was returned from + * stack_depot_save(). + * @buf: Pointer to the print buffer + * + * @size: Size of the print buffer + * + * @spaces: Number of leading spaces to print + * + * Return: Number of bytes printed. + */ +int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, + int spaces) +{ + unsigned long *entries; + unsigned int nr_entries; + + nr_entries = stack_depot_fetch(handle, &entries); + return nr_entries ? stack_trace_snprint(buf, size, entries, nr_entries, + spaces) : 0; +} +EXPORT_SYMBOL_GPL(stack_depot_snprint); + +/** + * stack_depot_print - print stack entries from a depot + * + * @stack: Stack depot handle which was returned from + * stack_depot_save(). + * + */ +void stack_depot_print(depot_stack_handle_t stack) +{ + unsigned long *entries; + unsigned int nr_entries; + + nr_entries = stack_depot_fetch(stack, &entries); + if (nr_entries > 0) + stack_trace_print(entries, nr_entries, 0); +} +EXPORT_SYMBOL_GPL(stack_depot_print); + +/** * stack_depot_fetch - Fetch stack entries from a depot * * @handle: Stack depot handle which was returned from @@ -198,9 +269,23 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries) { union handle_parts parts = { .handle = handle }; - void *slab = stack_slabs[parts.slabindex]; + void *slab; size_t offset = parts.offset << STACK_ALLOC_ALIGN; - struct stack_record *stack = slab + offset; + struct stack_record *stack; + + *entries = NULL; + if (!handle) + return 0; + + if (parts.slabindex > depot_index) { + WARN(1, "slab index %d out of bounds (%d) for stack id %08x\n", + parts.slabindex, depot_index, handle); + return 0; + } + slab = stack_slabs[parts.slabindex]; + if (!slab) + return 0; + stack = slab + offset; *entries = stack->entries; return stack->size; @@ -208,17 +293,28 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, EXPORT_SYMBOL_GPL(stack_depot_fetch); /** - * stack_depot_save - Save a stack trace from an array + * __stack_depot_save - Save a stack trace from an array * * @entries: Pointer to storage array * @nr_entries: Size of the storage array * @alloc_flags: Allocation gfp flags + * @can_alloc: Allocate stack slabs (increased chance of failure if false) * - * Return: The handle of the stack struct stored in depot + * Saves a stack trace from @entries array of size @nr_entries. If @can_alloc is + * %true, is allowed to replenish the stack slab pool in case no space is left + * (allocates using GFP flags of @alloc_flags). If @can_alloc is %false, avoids + * any allocations and will fail if no space is left to store the stack trace. + * + * Context: Any context, but setting @can_alloc to %false is required if + * alloc_pages() cannot be used from the current context. Currently + * this is the case from contexts where neither %GFP_ATOMIC nor + * %GFP_NOWAIT can be used (NMI, raw_spin_lock). + * + * Return: The handle of the stack struct stored in depot, 0 on failure. */ -depot_stack_handle_t stack_depot_save(unsigned long *entries, - unsigned int nr_entries, - gfp_t alloc_flags) +depot_stack_handle_t __stack_depot_save(unsigned long *entries, + unsigned int nr_entries, + gfp_t alloc_flags, bool can_alloc) { struct stack_record *found = NULL, **bucket; depot_stack_handle_t retval = 0; @@ -227,7 +323,7 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, unsigned long flags; u32 hash; - if (unlikely(nr_entries == 0)) + if (unlikely(nr_entries == 0) || stack_depot_disable) goto fast_exit; hash = hash_stack(entries, nr_entries); @@ -251,7 +347,7 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, * The smp_load_acquire() here pairs with smp_store_release() to * |next_slab_inited| in depot_alloc_stack() and init_stack_slab(). */ - if (unlikely(!smp_load_acquire(&next_slab_inited))) { + if (unlikely(can_alloc && !smp_load_acquire(&next_slab_inited))) { /* * Zero out zone modifiers, as we don't have specific zone * requirements. Keep the flags related to allocation in atomic @@ -265,13 +361,12 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, prealloc = page_address(page); } - spin_lock_irqsave(&depot_lock, flags); + raw_spin_lock_irqsave(&depot_lock, flags); found = find_stack(*bucket, entries, nr_entries, hash); if (!found) { - struct stack_record *new = - depot_alloc_stack(entries, nr_entries, - hash, &prealloc, alloc_flags); + struct stack_record *new = depot_alloc_stack(entries, nr_entries, hash, &prealloc); + if (new) { new->next = *bucket; /* @@ -289,7 +384,7 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, WARN_ON(!init_stack_slab(&prealloc)); } - spin_unlock_irqrestore(&depot_lock, flags); + raw_spin_unlock_irqrestore(&depot_lock, flags); exit: if (prealloc) { /* Nobody used this memory, ok to free it. */ @@ -300,4 +395,24 @@ exit: fast_exit: return retval; } +EXPORT_SYMBOL_GPL(__stack_depot_save); + +/** + * stack_depot_save - Save a stack trace from an array + * + * @entries: Pointer to storage array + * @nr_entries: Size of the storage array + * @alloc_flags: Allocation gfp flags + * + * Context: Contexts where allocations via alloc_pages() are allowed. + * See __stack_depot_save() for more details. + * + * Return: The handle of the stack struct stored in depot, 0 on failure. + */ +depot_stack_handle_t stack_depot_save(unsigned long *entries, + unsigned int nr_entries, + gfp_t alloc_flags) +{ + return __stack_depot_save(entries, nr_entries, alloc_flags, true); +} EXPORT_SYMBOL_GPL(stack_depot_save); diff --git a/lib/string.c b/lib/string.c index 08ec58cc673b..485777c9da83 100644 --- a/lib/string.c +++ b/lib/string.c @@ -6,20 +6,15 @@ */ /* - * stupid library routines.. The optimized versions should generally be found - * as inline code in <asm-xx/string.h> + * This file should be used only for "library" routines that may have + * alternative implementations on specific architectures (generally + * found in <asm-xx/string.h>), or get overloaded by FORTIFY_SOURCE. + * (Specifically, this file is built with __NO_FORTIFY.) * - * These are buggy as well.. - * - * * Fri Jun 25 1999, Ingo Oeser <ioe@informatik.tu-chemnitz.de> - * - Added strsep() which will replace strtok() soon (because strsep() is - * reentrant and should be faster). Use only strsep() in new code, please. - * - * * Sat Feb 09 2002, Jason Thomas <jason@topic.com.au>, - * Matthew Hawkins <matt@mh.dropbear.id.au> - * - Kissed strtok() goodbye + * Other helper functions should live in string_helpers.c. */ +#define __NO_FORTIFY #include <linux/types.h> #include <linux/string.h> #include <linux/ctype.h> @@ -29,6 +24,7 @@ #include <linux/errno.h> #include <linux/slab.h> +#include <asm/unaligned.h> #include <asm/byteorder.h> #include <asm/word-at-a-time.h> #include <asm/page.h> @@ -85,7 +81,6 @@ EXPORT_SYMBOL(strcasecmp); * @dest: Where to copy the string to * @src: Where to copy the string from */ -#undef strcpy char *strcpy(char *dest, const char *src) { char *tmp = dest; @@ -239,38 +234,28 @@ EXPORT_SYMBOL(strscpy); #endif /** - * strscpy_pad() - Copy a C-string into a sized buffer - * @dest: Where to copy the string to - * @src: Where to copy the string from - * @count: Size of destination buffer + * stpcpy - copy a string from src to dest returning a pointer to the new end + * of dest, including src's %NUL-terminator. May overrun dest. + * @dest: pointer to end of string being copied into. Must be large enough + * to receive copy. + * @src: pointer to the beginning of string being copied from. Must not overlap + * dest. * - * Copy the string, or as much of it as fits, into the dest buffer. The - * behavior is undefined if the string buffers overlap. The destination - * buffer is always %NUL terminated, unless it's zero-sized. - * - * If the source string is shorter than the destination buffer, zeros - * the tail of the destination buffer. - * - * For full explanation of why you may want to consider using the - * 'strscpy' functions please see the function docstring for strscpy(). - * - * Returns: - * * The number of characters copied (not including the trailing %NUL) - * * -E2BIG if count is 0 or @src was truncated. + * stpcpy differs from strcpy in a key way: the return value is a pointer + * to the new %NUL-terminating character in @dest. (For strcpy, the return + * value is a pointer to the start of @dest). This interface is considered + * unsafe as it doesn't perform bounds checking of the inputs. As such it's + * not recommended for usage. Instead, its definition is provided in case + * the compiler lowers other libcalls to stpcpy. */ -ssize_t strscpy_pad(char *dest, const char *src, size_t count) +char *stpcpy(char *__restrict__ dest, const char *__restrict__ src); +char *stpcpy(char *__restrict__ dest, const char *__restrict__ src) { - ssize_t written; - - written = strscpy(dest, src, count); - if (written < 0 || written == count - 1) - return written; - - memset(dest + written + 1, 0, count - written - 1); - - return written; + while ((*dest++ = *src++) != '\0') + /* nothing */; + return --dest; } -EXPORT_SYMBOL(strscpy_pad); +EXPORT_SYMBOL(stpcpy); #ifndef __HAVE_ARCH_STRCAT /** @@ -278,7 +263,6 @@ EXPORT_SYMBOL(strscpy_pad); * @dest: The string to be appended to * @src: The string to append to it */ -#undef strcat char *strcat(char *dest, const char *src) { char *tmp = dest; @@ -354,7 +338,6 @@ EXPORT_SYMBOL(strlcat); * @cs: One string * @ct: Another string */ -#undef strcmp int strcmp(const char *cs, const char *ct) { unsigned char c1, c2; @@ -434,6 +417,23 @@ char *strchrnul(const char *s, int c) EXPORT_SYMBOL(strchrnul); #endif +/** + * strnchrnul - Find and return a character in a length limited string, + * or end of string + * @s: The string to be searched + * @count: The number of characters to be searched + * @c: The character to search for + * + * Returns pointer to the first occurrence of 'c' in s. If c is not found, + * then return a pointer to the last character of the string. + */ +char *strnchrnul(const char *s, size_t count, int c) +{ + while (count-- && *s && *s != (char)c) + s++; + return (char *)s; +} + #ifndef __HAVE_ARCH_STRRCHR /** * strrchr - Find the last occurrence of a character in a string @@ -475,46 +475,6 @@ char *strnchr(const char *s, size_t count, int c) EXPORT_SYMBOL(strnchr); #endif -/** - * skip_spaces - Removes leading whitespace from @str. - * @str: The string to be stripped. - * - * Returns a pointer to the first non-whitespace character in @str. - */ -char *skip_spaces(const char *str) -{ - while (isspace(*str)) - ++str; - return (char *)str; -} -EXPORT_SYMBOL(skip_spaces); - -/** - * strim - Removes leading and trailing whitespace from @s. - * @s: The string to be stripped. - * - * Note that the first trailing whitespace is replaced with a %NUL-terminator - * in the given string @s. Returns a pointer to the first non-whitespace - * character in @s. - */ -char *strim(char *s) -{ - size_t size; - char *end; - - size = strlen(s); - if (!size) - return s; - - end = s + size - 1; - while (end >= s && isspace(*end)) - end--; - *(end + 1) = '\0'; - - return skip_spaces(s); -} -EXPORT_SYMBOL(strim); - #ifndef __HAVE_ARCH_STRLEN /** * strlen - Find the length of a string @@ -649,85 +609,6 @@ char *strsep(char **s, const char *ct) EXPORT_SYMBOL(strsep); #endif -/** - * sysfs_streq - return true if strings are equal, modulo trailing newline - * @s1: one string - * @s2: another string - * - * This routine returns true iff two strings are equal, treating both - * NUL and newline-then-NUL as equivalent string terminations. It's - * geared for use with sysfs input strings, which generally terminate - * with newlines but are compared against values without newlines. - */ -bool sysfs_streq(const char *s1, const char *s2) -{ - while (*s1 && *s1 == *s2) { - s1++; - s2++; - } - - if (*s1 == *s2) - return true; - if (!*s1 && *s2 == '\n' && !s2[1]) - return true; - if (*s1 == '\n' && !s1[1] && !*s2) - return true; - return false; -} -EXPORT_SYMBOL(sysfs_streq); - -/** - * match_string - matches given string in an array - * @array: array of strings - * @n: number of strings in the array or -1 for NULL terminated arrays - * @string: string to match with - * - * Return: - * index of a @string in the @array if matches, or %-EINVAL otherwise. - */ -int match_string(const char * const *array, size_t n, const char *string) -{ - int index; - const char *item; - - for (index = 0; index < n; index++) { - item = array[index]; - if (!item) - break; - if (!strcmp(item, string)) - return index; - } - - return -EINVAL; -} -EXPORT_SYMBOL(match_string); - -/** - * __sysfs_match_string - matches given string in an array - * @array: array of strings - * @n: number of strings in the array or -1 for NULL terminated arrays - * @str: string to match with - * - * Returns index of @str in the @array or -EINVAL, just like match_string(). - * Uses sysfs_streq instead of strcmp for matching. - */ -int __sysfs_match_string(const char * const *array, size_t n, const char *str) -{ - const char *item; - int index; - - for (index = 0; index < n; index++) { - item = array[index]; - if (!item) - break; - if (sysfs_streq(item, str)) - return index; - } - - return -EINVAL; -} -EXPORT_SYMBOL(__sysfs_match_string); - #ifndef __HAVE_ARCH_MEMSET /** * memset - Fill a region of memory with the given value @@ -881,6 +762,21 @@ __visible int memcmp(const void *cs, const void *ct, size_t count) const unsigned char *su1, *su2; int res = 0; +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (count >= sizeof(unsigned long)) { + const unsigned long *u1 = cs; + const unsigned long *u2 = ct; + do { + if (get_unaligned(u1) != get_unaligned(u2)) + break; + u1++; + u2++; + count -= sizeof(unsigned long); + } while (count >= sizeof(unsigned long)); + cs = u1; + ct = u2; + } +#endif for (su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--) if ((res = *su1 - *su2) != 0) break; @@ -901,7 +797,6 @@ EXPORT_SYMBOL(memcmp); * while this particular implementation is a simple (tail) call to memcmp, do * not rely on anything but whether the return value is zero or non-zero. */ -#undef bcmp int bcmp(const void *a, const void *b, size_t len) { return memcmp(a, b, len); @@ -924,7 +819,7 @@ void *memscan(void *addr, int c, size_t size) unsigned char *p = addr; while (size) { - if (*p == c) + if (*p == (unsigned char)c) return (void *)p; p++; size--; @@ -1072,27 +967,3 @@ void *memchr_inv(const void *start, int c, size_t bytes) return check_bytes8(start, value, bytes % 8); } EXPORT_SYMBOL(memchr_inv); - -/** - * strreplace - Replace all occurrences of character in string. - * @s: The string to operate on. - * @old: The character being replaced. - * @new: The character @old is replaced with. - * - * Returns pointer to the nul byte at the end of @s. - */ -char *strreplace(char *s, char old, char new) -{ - for (; *s; ++s) - if (*s == old) - *s = new; - return s; -} -EXPORT_SYMBOL(strreplace); - -void fortify_panic(const char *name) -{ - pr_emerg("detected buffer overflow in %s\n", name); - BUG(); -} -EXPORT_SYMBOL(fortify_panic); diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 963050c0283e..d5d008f5b1d9 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -361,6 +361,9 @@ static bool escape_special(unsigned char c, char **dst, char *end) case '\e': to = 'e'; break; + case '"': + to = '"'; + break; default: return false; } @@ -452,18 +455,20 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * The process of escaping byte buffer includes several parts. They are applied * in the following sequence. * - * 1. The character is matched to the printable class, if asked, and in - * case of match it passes through to the output. - * 2. The character is not matched to the one from @only string and thus + * 1. The character is not matched to the one from @only string and thus * must go as-is to the output. - * 3. The character is checked if it falls into the class given by @flags. + * 2. The character is matched to the printable and ASCII classes, if asked, + * and in case of match it passes through to the output. + * 3. The character is matched to the printable or ASCII class, if asked, + * and in case of match it passes through to the output. + * 4. The character is checked if it falls into the class given by @flags. * %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any * character. Note that they actually can't go together, otherwise * %ESCAPE_HEX will be ignored. * * Caller must provide valid source and destination pointers. Be aware that * destination buffer will not be NULL-terminated, thus caller have to append - * it if needs. The supported flags are:: + * it if needs. The supported flags are:: * * %ESCAPE_SPACE: (special white space, not space itself) * '\f' - form feed @@ -472,6 +477,7 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * '\t' - horizontal tab * '\v' - vertical tab * %ESCAPE_SPECIAL: + * '\"' - double quote * '\\' - backslash * '\a' - alert (BEL) * '\e' - escape @@ -482,11 +488,27 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * %ESCAPE_ANY: * all previous together * %ESCAPE_NP: - * escape only non-printable characters (checked by isprint) + * escape only non-printable characters, checked by isprint() * %ESCAPE_ANY_NP: * all previous together * %ESCAPE_HEX: * '\xHH' - byte with hexadecimal value HH (2 digits) + * %ESCAPE_NA: + * escape only non-ascii characters, checked by isascii() + * %ESCAPE_NAP: + * escape only non-printable or non-ascii characters + * %ESCAPE_APPEND: + * append characters from @only to be escaped by the given classes + * + * %ESCAPE_APPEND would help to pass additional characters to the escaped, when + * one of %ESCAPE_NP, %ESCAPE_NA, or %ESCAPE_NAP is provided. + * + * One notable caveat, the %ESCAPE_NAP, %ESCAPE_NP and %ESCAPE_NA have the + * higher priority than the rest of the flags (%ESCAPE_NAP is the highest). + * It doesn't make much sense to use either of them without %ESCAPE_OCTAL + * or %ESCAPE_HEX, because they cover most of the other character classes. + * %ESCAPE_NAP can utilize %ESCAPE_SPACE or %ESCAPE_SPECIAL in addition to + * the above. * * Return: * The total size of the escaped output that would be generated for @@ -500,67 +522,69 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, char *p = dst; char *end = p + osz; bool is_dict = only && *only; + bool is_append = flags & ESCAPE_APPEND; while (isz--) { unsigned char c = *src++; + bool in_dict = is_dict && strchr(only, c); /* * Apply rules in the following sequence: - * - the character is printable, when @flags has - * %ESCAPE_NP bit set * - the @only string is supplied and does not contain a * character under question + * - the character is printable and ASCII, when @flags has + * %ESCAPE_NAP bit set + * - the character is printable, when @flags has + * %ESCAPE_NP bit set + * - the character is ASCII, when @flags has + * %ESCAPE_NA bit set * - the character doesn't fall into a class of symbols * defined by given @flags * In these cases we just pass through a character to the * output buffer. + * + * When %ESCAPE_APPEND is passed, the characters from @only + * have been excluded from the %ESCAPE_NAP, %ESCAPE_NP, and + * %ESCAPE_NA cases. */ - if ((flags & ESCAPE_NP && isprint(c)) || - (is_dict && !strchr(only, c))) { - /* do nothing */ - } else { - if (flags & ESCAPE_SPACE && escape_space(c, &p, end)) - continue; + if (!(is_append || in_dict) && is_dict && + escape_passthrough(c, &p, end)) + continue; - if (flags & ESCAPE_SPECIAL && escape_special(c, &p, end)) - continue; + if (!(is_append && in_dict) && isascii(c) && isprint(c) && + flags & ESCAPE_NAP && escape_passthrough(c, &p, end)) + continue; - if (flags & ESCAPE_NULL && escape_null(c, &p, end)) - continue; + if (!(is_append && in_dict) && isprint(c) && + flags & ESCAPE_NP && escape_passthrough(c, &p, end)) + continue; - /* ESCAPE_OCTAL and ESCAPE_HEX always go last */ - if (flags & ESCAPE_OCTAL && escape_octal(c, &p, end)) - continue; + if (!(is_append && in_dict) && isascii(c) && + flags & ESCAPE_NA && escape_passthrough(c, &p, end)) + continue; - if (flags & ESCAPE_HEX && escape_hex(c, &p, end)) - continue; - } + if (flags & ESCAPE_SPACE && escape_space(c, &p, end)) + continue; - escape_passthrough(c, &p, end); - } + if (flags & ESCAPE_SPECIAL && escape_special(c, &p, end)) + continue; - return p - dst; -} -EXPORT_SYMBOL(string_escape_mem); + if (flags & ESCAPE_NULL && escape_null(c, &p, end)) + continue; -int string_escape_mem_ascii(const char *src, size_t isz, char *dst, - size_t osz) -{ - char *p = dst; - char *end = p + osz; + /* ESCAPE_OCTAL and ESCAPE_HEX always go last */ + if (flags & ESCAPE_OCTAL && escape_octal(c, &p, end)) + continue; - while (isz--) { - unsigned char c = *src++; + if (flags & ESCAPE_HEX && escape_hex(c, &p, end)) + continue; - if (!isprint(c) || !isascii(c) || c == '"' || c == '\\') - escape_hex(c, &p, end); - else - escape_passthrough(c, &p, end); + escape_passthrough(c, &p, end); } return p - dst; } -EXPORT_SYMBOL(string_escape_mem_ascii); +EXPORT_SYMBOL(string_escape_mem); /* * Return an allocated string that has been escaped of special characters @@ -649,3 +673,241 @@ char *kstrdup_quotable_file(struct file *file, gfp_t gfp) return pathname; } EXPORT_SYMBOL_GPL(kstrdup_quotable_file); + +/** + * kfree_strarray - free a number of dynamically allocated strings contained + * in an array and the array itself + * + * @array: Dynamically allocated array of strings to free. + * @n: Number of strings (starting from the beginning of the array) to free. + * + * Passing a non-NULL @array and @n == 0 as well as NULL @array are valid + * use-cases. If @array is NULL, the function does nothing. + */ +void kfree_strarray(char **array, size_t n) +{ + unsigned int i; + + if (!array) + return; + + for (i = 0; i < n; i++) + kfree(array[i]); + kfree(array); +} +EXPORT_SYMBOL_GPL(kfree_strarray); + +/** + * strscpy_pad() - Copy a C-string into a sized buffer + * @dest: Where to copy the string to + * @src: Where to copy the string from + * @count: Size of destination buffer + * + * Copy the string, or as much of it as fits, into the dest buffer. The + * behavior is undefined if the string buffers overlap. The destination + * buffer is always %NUL terminated, unless it's zero-sized. + * + * If the source string is shorter than the destination buffer, zeros + * the tail of the destination buffer. + * + * For full explanation of why you may want to consider using the + * 'strscpy' functions please see the function docstring for strscpy(). + * + * Returns: + * * The number of characters copied (not including the trailing %NUL) + * * -E2BIG if count is 0 or @src was truncated. + */ +ssize_t strscpy_pad(char *dest, const char *src, size_t count) +{ + ssize_t written; + + written = strscpy(dest, src, count); + if (written < 0 || written == count - 1) + return written; + + memset(dest + written + 1, 0, count - written - 1); + + return written; +} +EXPORT_SYMBOL(strscpy_pad); + +/** + * skip_spaces - Removes leading whitespace from @str. + * @str: The string to be stripped. + * + * Returns a pointer to the first non-whitespace character in @str. + */ +char *skip_spaces(const char *str) +{ + while (isspace(*str)) + ++str; + return (char *)str; +} +EXPORT_SYMBOL(skip_spaces); + +/** + * strim - Removes leading and trailing whitespace from @s. + * @s: The string to be stripped. + * + * Note that the first trailing whitespace is replaced with a %NUL-terminator + * in the given string @s. Returns a pointer to the first non-whitespace + * character in @s. + */ +char *strim(char *s) +{ + size_t size; + char *end; + + size = strlen(s); + if (!size) + return s; + + end = s + size - 1; + while (end >= s && isspace(*end)) + end--; + *(end + 1) = '\0'; + + return skip_spaces(s); +} +EXPORT_SYMBOL(strim); + +/** + * sysfs_streq - return true if strings are equal, modulo trailing newline + * @s1: one string + * @s2: another string + * + * This routine returns true iff two strings are equal, treating both + * NUL and newline-then-NUL as equivalent string terminations. It's + * geared for use with sysfs input strings, which generally terminate + * with newlines but are compared against values without newlines. + */ +bool sysfs_streq(const char *s1, const char *s2) +{ + while (*s1 && *s1 == *s2) { + s1++; + s2++; + } + + if (*s1 == *s2) + return true; + if (!*s1 && *s2 == '\n' && !s2[1]) + return true; + if (*s1 == '\n' && !s1[1] && !*s2) + return true; + return false; +} +EXPORT_SYMBOL(sysfs_streq); + +/** + * match_string - matches given string in an array + * @array: array of strings + * @n: number of strings in the array or -1 for NULL terminated arrays + * @string: string to match with + * + * This routine will look for a string in an array of strings up to the + * n-th element in the array or until the first NULL element. + * + * Historically the value of -1 for @n, was used to search in arrays that + * are NULL terminated. However, the function does not make a distinction + * when finishing the search: either @n elements have been compared OR + * the first NULL element was found. + * + * Return: + * index of a @string in the @array if matches, or %-EINVAL otherwise. + */ +int match_string(const char * const *array, size_t n, const char *string) +{ + int index; + const char *item; + + for (index = 0; index < n; index++) { + item = array[index]; + if (!item) + break; + if (!strcmp(item, string)) + return index; + } + + return -EINVAL; +} +EXPORT_SYMBOL(match_string); + +/** + * __sysfs_match_string - matches given string in an array + * @array: array of strings + * @n: number of strings in the array or -1 for NULL terminated arrays + * @str: string to match with + * + * Returns index of @str in the @array or -EINVAL, just like match_string(). + * Uses sysfs_streq instead of strcmp for matching. + * + * This routine will look for a string in an array of strings up to the + * n-th element in the array or until the first NULL element. + * + * Historically the value of -1 for @n, was used to search in arrays that + * are NULL terminated. However, the function does not make a distinction + * when finishing the search: either @n elements have been compared OR + * the first NULL element was found. + */ +int __sysfs_match_string(const char * const *array, size_t n, const char *str) +{ + const char *item; + int index; + + for (index = 0; index < n; index++) { + item = array[index]; + if (!item) + break; + if (sysfs_streq(item, str)) + return index; + } + + return -EINVAL; +} +EXPORT_SYMBOL(__sysfs_match_string); + +/** + * strreplace - Replace all occurrences of character in string. + * @s: The string to operate on. + * @old: The character being replaced. + * @new: The character @old is replaced with. + * + * Returns pointer to the nul byte at the end of @s. + */ +char *strreplace(char *s, char old, char new) +{ + for (; *s; ++s) + if (*s == old) + *s = new; + return s; +} +EXPORT_SYMBOL(strreplace); + +/** + * memcpy_and_pad - Copy one buffer to another with padding + * @dest: Where to copy to + * @dest_len: The destination buffer size + * @src: Where to copy from + * @count: The number of bytes to copy + * @pad: Character to use for padding if space is left in destination. + */ +void memcpy_and_pad(void *dest, size_t dest_len, const void *src, size_t count, + int pad) +{ + if (dest_len > count) { + memcpy(dest, src, count); + memset(dest + count, pad, dest_len - count); + } else { + memcpy(dest, src, dest_len); + } +} +EXPORT_SYMBOL(memcpy_and_pad); + +#ifdef CONFIG_FORTIFY_SOURCE +void fortify_panic(const char *name) +{ + pr_emerg("detected buffer overflow in %s\n", name); + BUG(); +} +EXPORT_SYMBOL(fortify_panic); +#endif /* CONFIG_FORTIFY_SOURCE */ diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c index dccb95af6003..122d8d0e253c 100644 --- a/lib/strncpy_from_user.c +++ b/lib/strncpy_from_user.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/compiler.h> #include <linux/export.h> +#include <linux/fault-inject-usercopy.h> #include <linux/kasan-checks.h> #include <linux/thread_info.h> #include <linux/uaccess.h> @@ -30,28 +31,36 @@ static inline long do_strncpy_from_user(char *dst, const char __user *src, const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS; unsigned long res = 0; - /* - * Truncate 'max' to the user-specified limit, so that - * we only have one limit we need to check in the loop - */ - if (max > count) - max = count; - if (IS_UNALIGNED(src, dst)) goto byte_at_a_time; while (max >= sizeof(unsigned long)) { - unsigned long c, data; + unsigned long c, data, mask; /* Fall back to byte-at-a-time if we get a page fault */ unsafe_get_user(c, (unsigned long __user *)(src+res), byte_at_a_time); - *(unsigned long *)(dst+res) = c; + /* + * Note that we mask out the bytes following the NUL. This is + * important to do because string oblivious code may read past + * the NUL. For those routines, we don't want to give them + * potentially random bytes after the NUL in `src`. + * + * One example of such code is BPF map keys. BPF treats map keys + * as an opaque set of bytes. Without the post-NUL mask, any BPF + * maps keyed by strings returned from strncpy_from_user() may + * have multiple entries for semantically identical strings. + */ if (has_zero(c, &data, &constants)) { data = prep_zero_mask(c, data, &constants); data = create_zero_mask(data); + mask = zero_bytemask(data); + *(unsigned long *)(dst+res) = c & mask; return res + find_zero(data); } + + *(unsigned long *)(dst+res) = c; + res += sizeof(unsigned long); max -= sizeof(unsigned long); } @@ -105,6 +114,9 @@ long strncpy_from_user(char *dst, const char __user *src, long count) { unsigned long max_addr, src_addr; + might_fault(); + if (should_fail_usercopy()) + return -EFAULT; if (unlikely(count <= 0)) return 0; @@ -114,11 +126,18 @@ long strncpy_from_user(char *dst, const char __user *src, long count) unsigned long max = max_addr - src_addr; long retval; + /* + * Truncate 'max' to the user-specified limit, so that + * we only have one limit we need to check in the loop + */ + if (max > count) + max = count; + kasan_check_write(dst, count); check_object_size(dst, count, false); - if (user_access_begin(src, max)) { + if (user_read_access_begin(src, max)) { retval = do_strncpy_from_user(dst, src, count, max); - user_access_end(); + user_read_access_end(); return retval; } } diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c index 6c0005d5dd5c..1616710b8a82 100644 --- a/lib/strnlen_user.c +++ b/lib/strnlen_user.c @@ -27,13 +27,6 @@ static inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long c; /* - * Truncate 'max' to the user-specified limit, so that - * we only have one limit we need to check in the loop - */ - if (max > count) - max = count; - - /* * Do everything aligned. But that means that we * need to also expand the maximum.. */ @@ -109,9 +102,16 @@ long strnlen_user(const char __user *str, long count) unsigned long max = max_addr - src_addr; long retval; - if (user_access_begin(str, max)) { + /* + * Truncate 'max' to the user-specified limit, so that + * we only have one limit we need to check in the loop + */ + if (max > count) + max = count; + + if (user_read_access_begin(str, max)) { retval = do_strnlen_user(str, count, max); - user_access_end(); + user_read_access_end(); return retval; } } diff --git a/lib/syscall.c b/lib/syscall.c index fb328e7ccb08..006e256d2264 100644 --- a/lib/syscall.c +++ b/lib/syscall.c @@ -7,6 +7,7 @@ static int collect_syscall(struct task_struct *target, struct syscall_info *info) { + unsigned long args[6] = { }; struct pt_regs *regs; if (!try_get_task_stack(target)) { @@ -27,8 +28,14 @@ static int collect_syscall(struct task_struct *target, struct syscall_info *info info->data.nr = syscall_get_nr(target, regs); if (info->data.nr != -1L) - syscall_get_arguments(target, regs, - (unsigned long *)&info->data.args[0]); + syscall_get_arguments(target, regs, args); + + info->data.args[0] = args[0]; + info->data.args[1] = args[1]; + info->data.args[2] = args[2]; + info->data.args[3] = args[3]; + info->data.args[4] = args[4]; + info->data.args[5] = args[5]; put_task_stack(target); return 0; @@ -44,7 +51,7 @@ static int collect_syscall(struct task_struct *target, struct syscall_info *info * .data.instruction_pointer - filled with user PC * * If @target is blocked in a system call, returns zero with @info.data.nr - * set to the the call's number and @info.data.args filled in with its + * set to the call's number and @info.data.args filled in with its * arguments. Registers not used for system call arguments may not be available * and it is not kosher to use &struct user_regset calls while the system * call is still in progress. Note we may get this result if @target @@ -61,13 +68,13 @@ static int collect_syscall(struct task_struct *target, struct syscall_info *info */ int task_current_syscall(struct task_struct *target, struct syscall_info *info) { - long state; unsigned long ncsw; + unsigned int state; if (target == current) return collect_syscall(target, info); - state = target->state; + state = READ_ONCE(target->__state); if (unlikely(!state)) return -EAGAIN; diff --git a/lib/test-string_helpers.c b/lib/test-string_helpers.c index 25b5cbfb7615..437d8e6b7cb1 100644 --- a/lib/test-string_helpers.c +++ b/lib/test-string_helpers.c @@ -19,7 +19,7 @@ static __init bool test_string_check_buf(const char *name, unsigned int flags, if (q_real == q_test && !memcmp(out_test, out_real, q_test)) return true; - pr_warn("Test '%s' failed: flags = %u\n", name, flags); + pr_warn("Test '%s' failed: flags = %#x\n", name, flags); print_hex_dump(KERN_WARNING, "Input: ", DUMP_PREFIX_NONE, 16, 1, in, p, true); @@ -136,31 +136,31 @@ static const struct test_string_2 escape0[] __initconst = {{ .flags = ESCAPE_SPACE | ESCAPE_HEX, },{ /* terminator */ - }}, + }} },{ .in = "\\h\\\"\a\e\\", .s1 = {{ - .out = "\\\\h\\\\\"\\a\\e\\\\", + .out = "\\\\h\\\\\\\"\\a\\e\\\\", .flags = ESCAPE_SPECIAL, },{ - .out = "\\\\\\150\\\\\\042\\a\\e\\\\", + .out = "\\\\\\150\\\\\\\"\\a\\e\\\\", .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL, },{ - .out = "\\\\\\x68\\\\\\x22\\a\\e\\\\", + .out = "\\\\\\x68\\\\\\\"\\a\\e\\\\", .flags = ESCAPE_SPECIAL | ESCAPE_HEX, },{ /* terminator */ - }}, + }} },{ .in = "\eb \\C\007\"\x90\r]", .s1 = {{ .out = "\eb \\C\007\"\x90\\r]", .flags = ESCAPE_SPACE, },{ - .out = "\\eb \\\\C\\a\"\x90\r]", + .out = "\\eb \\\\C\\a\\\"\x90\r]", .flags = ESCAPE_SPECIAL, },{ - .out = "\\eb \\\\C\\a\"\x90\\r]", + .out = "\\eb \\\\C\\a\\\"\x90\\r]", .flags = ESCAPE_SPACE | ESCAPE_SPECIAL, },{ .out = "\\033\\142\\040\\134\\103\\007\\042\\220\\015\\135", @@ -169,10 +169,10 @@ static const struct test_string_2 escape0[] __initconst = {{ .out = "\\033\\142\\040\\134\\103\\007\\042\\220\\r\\135", .flags = ESCAPE_SPACE | ESCAPE_OCTAL, },{ - .out = "\\e\\142\\040\\\\\\103\\a\\042\\220\\015\\135", + .out = "\\e\\142\\040\\\\\\103\\a\\\"\\220\\015\\135", .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL, },{ - .out = "\\e\\142\\040\\\\\\103\\a\\042\\220\\r\\135", + .out = "\\e\\142\\040\\\\\\103\\a\\\"\\220\\r\\135", .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_OCTAL, },{ .out = "\eb \\C\007\"\x90\r]", @@ -201,12 +201,26 @@ static const struct test_string_2 escape0[] __initconst = {{ .flags = ESCAPE_NP | ESCAPE_HEX, },{ /* terminator */ - }}, + }} +},{ + .in = "\007 \eb\"\x90\xCF\r", + .s1 = {{ + .out = "\007 \eb\"\\220\\317\r", + .flags = ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\\x90\\xcf\r", + .flags = ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_NA, + },{ + /* terminator */ + }} },{ /* terminator */ }}; -#define TEST_STRING_2_DICT_1 "b\\ \t\r" +#define TEST_STRING_2_DICT_1 "b\\ \t\r\xCF" static const struct test_string_2 escape1[] __initconst = {{ .in = "\f\\ \n\r\t\v", .s1 = {{ @@ -216,16 +230,40 @@ static const struct test_string_2 escape1[] __initconst = {{ .out = "\f\\x5c\\x20\n\\x0d\\x09\v", .flags = ESCAPE_HEX, },{ + .out = "\f\\134\\040\n\\015\\011\v", + .flags = ESCAPE_ANY | ESCAPE_APPEND, + },{ + .out = "\\014\\134\\040\\012\\015\\011\\013", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NAP, + },{ + .out = "\\x0c\\x5c\\x20\\x0a\\x0d\\x09\\x0b", + .flags = ESCAPE_HEX | ESCAPE_APPEND | ESCAPE_NAP, + },{ + .out = "\f\\134\\040\n\\015\\011\v", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NA, + },{ + .out = "\f\\x5c\\x20\n\\x0d\\x09\v", + .flags = ESCAPE_HEX | ESCAPE_APPEND | ESCAPE_NA, + },{ /* terminator */ - }}, + }} },{ - .in = "\\h\\\"\a\e\\", + .in = "\\h\\\"\a\xCF\e\\", .s1 = {{ - .out = "\\134h\\134\"\a\e\\134", + .out = "\\134h\\134\"\a\\317\e\\134", .flags = ESCAPE_OCTAL, },{ + .out = "\\134h\\134\"\a\\317\e\\134", + .flags = ESCAPE_ANY | ESCAPE_APPEND, + },{ + .out = "\\134h\\134\"\\007\\317\\033\\134", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NAP, + },{ + .out = "\\134h\\134\"\a\\317\e\\134", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NA, + },{ /* terminator */ - }}, + }} },{ .in = "\eb \\C\007\"\x90\r]", .s1 = {{ @@ -233,11 +271,115 @@ static const struct test_string_2 escape1[] __initconst = {{ .flags = ESCAPE_OCTAL, },{ /* terminator */ - }}, + }} +},{ + .in = "\007 \eb\"\x90\xCF\r", + .s1 = {{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPACE | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPECIAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_SPACE | ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_ANY | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_SPACE | ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NA, + },{ + /* terminator */ + }} +},{ + .in = "\007 \eb\"\x90\xCF\r", + .s1 = {{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\xCF\\r", + .flags = ESCAPE_SPACE | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPECIAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\xCF\\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\\015", + .flags = ESCAPE_OCTAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\\r", + .flags = ESCAPE_SPACE | ESCAPE_OCTAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\\015", + .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_ANY | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\x0d", + .flags = ESCAPE_HEX | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\r", + .flags = ESCAPE_SPACE | ESCAPE_HEX | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\x0d", + .flags = ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NAP, + },{ + /* terminator */ + }} },{ /* terminator */ }}; +static const struct test_string strings_upper[] __initconst = { + { + .in = "abcdefgh1234567890test", + .out = "ABCDEFGH1234567890TEST", + }, + { + .in = "abCdeFgH1234567890TesT", + .out = "ABCDEFGH1234567890TEST", + }, +}; + +static const struct test_string strings_lower[] __initconst = { + { + .in = "ABCDEFGH1234567890TEST", + .out = "abcdefgh1234567890test", + }, + { + .in = "abCdeFgH1234567890TesT", + .out = "abcdefgh1234567890test", + }, +}; + static __init const char *test_string_find_match(const struct test_string_2 *s2, unsigned int flags) { @@ -268,7 +410,7 @@ test_string_escape_overflow(const char *in, int p, unsigned int flags, const cha q_real = string_escape_mem(in, p, NULL, 0, flags, esc); if (q_real != q_test) - pr_warn("Test '%s' failed: flags = %u, osz = 0, expected %d, got %d\n", + pr_warn("Test '%s' failed: flags = %#x, osz = 0, expected %d, got %d\n", name, flags, q_test, q_real); } @@ -293,8 +435,13 @@ static __init void test_string_escape(const char *name, /* NULL injection */ if (flags & ESCAPE_NULL) { in[p++] = '\0'; - out_test[q_test++] = '\\'; - out_test[q_test++] = '0'; + /* '\0' passes isascii() test */ + if (flags & ESCAPE_NA && !(flags & ESCAPE_APPEND && esc)) { + out_test[q_test++] = '\0'; + } else { + out_test[q_test++] = '\\'; + out_test[q_test++] = '0'; + } } /* Don't try strings that have no output */ @@ -390,27 +537,72 @@ static __init void test_string_get_size(void) test_string_get_size_one(4096, U64_MAX, "75.6 ZB", "64.0 ZiB"); } +static void __init test_string_upper_lower(void) +{ + char *dst; + int i; + + for (i = 0; i < ARRAY_SIZE(strings_upper); i++) { + const char *s = strings_upper[i].in; + int len = strlen(strings_upper[i].in) + 1; + + dst = kmalloc(len, GFP_KERNEL); + if (!dst) + return; + + string_upper(dst, s); + if (memcmp(dst, strings_upper[i].out, len)) { + pr_warn("Test 'string_upper' failed : expected %s, got %s!\n", + strings_upper[i].out, dst); + kfree(dst); + return; + } + kfree(dst); + } + + for (i = 0; i < ARRAY_SIZE(strings_lower); i++) { + const char *s = strings_lower[i].in; + int len = strlen(strings_lower[i].in) + 1; + + dst = kmalloc(len, GFP_KERNEL); + if (!dst) + return; + + string_lower(dst, s); + if (memcmp(dst, strings_lower[i].out, len)) { + pr_warn("Test 'string_lower failed : : expected %s, got %s!\n", + strings_lower[i].out, dst); + kfree(dst); + return; + } + kfree(dst); + } +} + static int __init test_string_helpers_init(void) { unsigned int i; pr_info("Running tests...\n"); - for (i = 0; i < UNESCAPE_ANY + 1; i++) + for (i = 0; i < UNESCAPE_ALL_MASK + 1; i++) test_string_unescape("unescape", i, false); test_string_unescape("unescape inplace", get_random_int() % (UNESCAPE_ANY + 1), true); /* Without dictionary */ - for (i = 0; i < (ESCAPE_ANY_NP | ESCAPE_HEX) + 1; i++) + for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) test_string_escape("escape 0", escape0, i, TEST_STRING_2_DICT_0); /* With dictionary */ - for (i = 0; i < (ESCAPE_ANY_NP | ESCAPE_HEX) + 1; i++) + for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) test_string_escape("escape 1", escape1, i, TEST_STRING_2_DICT_1); /* Test string_get_size() */ test_string_get_size(); + /* Test string upper(), string_lower() */ + test_string_upper_lower(); + return -EINVAL; } module_init(test_string_helpers_init); diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index e14a15ac250b..d33fa5a61b95 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -16,10 +16,10 @@ #include "../tools/testing/selftests/kselftest_module.h" -static unsigned total_tests __initdata; -static unsigned failed_tests __initdata; +KSTM_MODULE_GLOBALS(); static char pbl_buffer[PAGE_SIZE] __initdata; +static char print_buf[PAGE_SIZE * 2] __initdata; static const unsigned long exp1[] __initconst = { BITMAP_FROM_U64(1), @@ -35,6 +35,8 @@ static const unsigned long exp1[] __initconst = { BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0xffffffff77777777ULL), BITMAP_FROM_U64(0), + BITMAP_FROM_U64(0x00008000), + BITMAP_FROM_U64(0x80000000), }; static const unsigned long exp2[] __initconst = { @@ -155,6 +157,20 @@ static bool __init __check_eq_clump8(const char *srcfile, unsigned int line, return true; } +static bool __init +__check_eq_str(const char *srcfile, unsigned int line, + const char *exp_str, const char *str, + unsigned int len) +{ + bool eq; + + eq = strncmp(exp_str, str, len) == 0; + if (!eq) + pr_err("[%s:%u] expected %s, got %s\n", srcfile, line, exp_str, str); + + return eq; +} + #define __expect_eq(suffix, ...) \ ({ \ int result = 0; \ @@ -172,6 +188,7 @@ static bool __init __check_eq_clump8(const char *srcfile, unsigned int line, #define expect_eq_pbl(...) __expect_eq(pbl, ##__VA_ARGS__) #define expect_eq_u32_array(...) __expect_eq(u32_array, ##__VA_ARGS__) #define expect_eq_clump8(...) __expect_eq(clump8, ##__VA_ARGS__) +#define expect_eq_str(...) __expect_eq(str, ##__VA_ARGS__) static void __init test_zero_clear(void) { @@ -275,26 +292,30 @@ static void __init test_copy(void) static void __init test_replace(void) { unsigned int nbits = 64; + unsigned int nlongs = DIV_ROUND_UP(nbits, BITS_PER_LONG); DECLARE_BITMAP(bmap, 1024); + BUILD_BUG_ON(EXP2_IN_BITS < nbits * 2); + bitmap_zero(bmap, 1024); - bitmap_replace(bmap, &exp2[0], &exp2[1], exp2_to_exp3_mask, nbits); + bitmap_replace(bmap, &exp2[0 * nlongs], &exp2[1 * nlongs], exp2_to_exp3_mask, nbits); expect_eq_bitmap(bmap, exp3_0_1, nbits); bitmap_zero(bmap, 1024); - bitmap_replace(bmap, &exp2[1], &exp2[0], exp2_to_exp3_mask, nbits); + bitmap_replace(bmap, &exp2[1 * nlongs], &exp2[0 * nlongs], exp2_to_exp3_mask, nbits); expect_eq_bitmap(bmap, exp3_1_0, nbits); bitmap_fill(bmap, 1024); - bitmap_replace(bmap, &exp2[0], &exp2[1], exp2_to_exp3_mask, nbits); + bitmap_replace(bmap, &exp2[0 * nlongs], &exp2[1 * nlongs], exp2_to_exp3_mask, nbits); expect_eq_bitmap(bmap, exp3_0_1, nbits); bitmap_fill(bmap, 1024); - bitmap_replace(bmap, &exp2[1], &exp2[0], exp2_to_exp3_mask, nbits); + bitmap_replace(bmap, &exp2[1 * nlongs], &exp2[0 * nlongs], exp2_to_exp3_mask, nbits); expect_eq_bitmap(bmap, exp3_1_0, nbits); } -#define PARSE_TIME 0x1 +#define PARSE_TIME 0x1 +#define NO_LEN 0x2 struct test_bitmap_parselist{ const int errno; @@ -331,15 +352,54 @@ static const struct test_bitmap_parselist parselist_tests[] __initconst = { {0, " , ,, , , ", &exp1[12 * step], 8, 0}, {0, " , ,, , , \n", &exp1[12 * step], 8, 0}, + {0, "0-0", &exp1[0], 32, 0}, + {0, "1-1", &exp1[1 * step], 32, 0}, + {0, "15-15", &exp1[13 * step], 32, 0}, + {0, "31-31", &exp1[14 * step], 32, 0}, + + {0, "0-0:0/1", &exp1[12 * step], 32, 0}, + {0, "0-0:1/1", &exp1[0], 32, 0}, + {0, "0-0:1/31", &exp1[0], 32, 0}, + {0, "0-0:31/31", &exp1[0], 32, 0}, + {0, "1-1:1/1", &exp1[1 * step], 32, 0}, + {0, "0-15:16/31", &exp1[2 * step], 32, 0}, + {0, "15-15:1/2", &exp1[13 * step], 32, 0}, + {0, "15-15:31/31", &exp1[13 * step], 32, 0}, + {0, "15-31:1/31", &exp1[13 * step], 32, 0}, + {0, "16-31:16/31", &exp1[3 * step], 32, 0}, + {0, "31-31:31/31", &exp1[14 * step], 32, 0}, + + {0, "N-N", &exp1[14 * step], 32, 0}, + {0, "0-0:1/N", &exp1[0], 32, 0}, + {0, "0-0:N/N", &exp1[0], 32, 0}, + {0, "0-15:16/N", &exp1[2 * step], 32, 0}, + {0, "15-15:N/N", &exp1[13 * step], 32, 0}, + {0, "15-N:1/N", &exp1[13 * step], 32, 0}, + {0, "16-N:16/N", &exp1[3 * step], 32, 0}, + {0, "N-N:N/N", &exp1[14 * step], 32, 0}, + + {0, "0-N:1/3,1-N:1/3,2-N:1/3", &exp1[8 * step], 32, 0}, + {0, "0-31:1/3,1-31:1/3,2-31:1/3", &exp1[8 * step], 32, 0}, + {0, "1-10:8/12,8-31:24/29,0-31:0/3", &exp1[9 * step], 32, 0}, + + {0, "all", &exp1[8 * step], 32, 0}, + {0, "0, 1, all, ", &exp1[8 * step], 32, 0}, + {0, "all:1/2", &exp1[4 * step], 32, 0}, + {0, "ALL:1/2", &exp1[4 * step], 32, 0}, + {-EINVAL, "al", NULL, 8, 0}, + {-EINVAL, "alll", NULL, 8, 0}, + {-EINVAL, "-1", NULL, 8, 0}, {-EINVAL, "-0", NULL, 8, 0}, {-EINVAL, "10-1", NULL, 8, 0}, - {-EINVAL, "0-31:", NULL, 8, 0}, - {-EINVAL, "0-31:0", NULL, 8, 0}, - {-EINVAL, "0-31:0/", NULL, 8, 0}, - {-EINVAL, "0-31:0/0", NULL, 8, 0}, - {-EINVAL, "0-31:1/0", NULL, 8, 0}, - {-EINVAL, "0-31:10/1", NULL, 8, 0}, + {-ERANGE, "8-8", NULL, 8, 0}, + {-ERANGE, "0-31", NULL, 8, 0}, + {-EINVAL, "0-31:", NULL, 32, 0}, + {-EINVAL, "0-31:0", NULL, 32, 0}, + {-EINVAL, "0-31:0/", NULL, 32, 0}, + {-EINVAL, "0-31:0/0", NULL, 32, 0}, + {-EINVAL, "0-31:1/0", NULL, 32, 0}, + {-EINVAL, "0-31:10/1", NULL, 32, 0}, {-EOVERFLOW, "0-98765432123456789:10/1", NULL, 8, 0}, {-EINVAL, "a-31", NULL, 8, 0}, @@ -348,66 +408,117 @@ static const struct test_bitmap_parselist parselist_tests[] __initconst = { {-EINVAL, "0-31:a/1", NULL, 8, 0}, {-EINVAL, "0-\n", NULL, 8, 0}, -#undef step }; -static void __init __test_bitmap_parselist(int is_user) +static void __init test_bitmap_parselist(void) { int i; int err; ktime_t time; DECLARE_BITMAP(bmap, 2048); - char *mode = is_user ? "_user" : ""; for (i = 0; i < ARRAY_SIZE(parselist_tests); i++) { #define ptest parselist_tests[i] - if (is_user) { - mm_segment_t orig_fs = get_fs(); - size_t len = strlen(ptest.in); - - set_fs(KERNEL_DS); - time = ktime_get(); - err = bitmap_parselist_user((__force const char __user *)ptest.in, len, - bmap, ptest.nbits); - time = ktime_get() - time; - set_fs(orig_fs); - } else { - time = ktime_get(); - err = bitmap_parselist(ptest.in, bmap, ptest.nbits); - time = ktime_get() - time; - } + time = ktime_get(); + err = bitmap_parselist(ptest.in, bmap, ptest.nbits); + time = ktime_get() - time; if (err != ptest.errno) { - pr_err("parselist%s: %d: input is %s, errno is %d, expected %d\n", - mode, i, ptest.in, err, ptest.errno); + pr_err("parselist: %d: input is %s, errno is %d, expected %d\n", + i, ptest.in, err, ptest.errno); continue; } if (!err && ptest.expected && !__bitmap_equal(bmap, ptest.expected, ptest.nbits)) { - pr_err("parselist%s: %d: input is %s, result is 0x%lx, expected 0x%lx\n", - mode, i, ptest.in, bmap[0], + pr_err("parselist: %d: input is %s, result is 0x%lx, expected 0x%lx\n", + i, ptest.in, bmap[0], *ptest.expected); continue; } if (ptest.flags & PARSE_TIME) - pr_err("parselist%s: %d: input is '%s' OK, Time: %llu\n", - mode, i, ptest.in, time); + pr_err("parselist: %d: input is '%s' OK, Time: %llu\n", + i, ptest.in, time); #undef ptest } } -static void __init test_bitmap_parselist(void) -{ - __test_bitmap_parselist(0); -} +static const unsigned long parse_test[] __initconst = { + BITMAP_FROM_U64(0), + BITMAP_FROM_U64(1), + BITMAP_FROM_U64(0xdeadbeef), + BITMAP_FROM_U64(0x100000000ULL), +}; -static void __init test_bitmap_parselist_user(void) +static const unsigned long parse_test2[] __initconst = { + BITMAP_FROM_U64(0x100000000ULL), BITMAP_FROM_U64(0xdeadbeef), + BITMAP_FROM_U64(0x100000000ULL), BITMAP_FROM_U64(0xbaadf00ddeadbeef), + BITMAP_FROM_U64(0x100000000ULL), BITMAP_FROM_U64(0x0badf00ddeadbeef), +}; + +static const struct test_bitmap_parselist parse_tests[] __initconst = { + {0, "", &parse_test[0 * step], 32, 0}, + {0, " ", &parse_test[0 * step], 32, 0}, + {0, "0", &parse_test[0 * step], 32, 0}, + {0, "0\n", &parse_test[0 * step], 32, 0}, + {0, "1", &parse_test[1 * step], 32, 0}, + {0, "deadbeef", &parse_test[2 * step], 32, 0}, + {0, "1,0", &parse_test[3 * step], 33, 0}, + {0, "deadbeef,\n,0,1", &parse_test[2 * step], 96, 0}, + + {0, "deadbeef,1,0", &parse_test2[0 * 2 * step], 96, 0}, + {0, "baadf00d,deadbeef,1,0", &parse_test2[1 * 2 * step], 128, 0}, + {0, "badf00d,deadbeef,1,0", &parse_test2[2 * 2 * step], 124, 0}, + {0, "badf00d,deadbeef,1,0", &parse_test2[2 * 2 * step], 124, NO_LEN}, + {0, " badf00d,deadbeef,1,0 ", &parse_test2[2 * 2 * step], 124, 0}, + {0, " , badf00d,deadbeef,1,0 , ", &parse_test2[2 * 2 * step], 124, 0}, + {0, " , badf00d, ,, ,,deadbeef,1,0 , ", &parse_test2[2 * 2 * step], 124, 0}, + + {-EINVAL, "goodfood,deadbeef,1,0", NULL, 128, 0}, + {-EOVERFLOW, "3,0", NULL, 33, 0}, + {-EOVERFLOW, "123badf00d,deadbeef,1,0", NULL, 128, 0}, + {-EOVERFLOW, "badf00d,deadbeef,1,0", NULL, 90, 0}, + {-EOVERFLOW, "fbadf00d,deadbeef,1,0", NULL, 95, 0}, + {-EOVERFLOW, "badf00d,deadbeef,1,0", NULL, 100, 0}, +#undef step +}; + +static void __init test_bitmap_parse(void) { - __test_bitmap_parselist(1); + int i; + int err; + ktime_t time; + DECLARE_BITMAP(bmap, 2048); + + for (i = 0; i < ARRAY_SIZE(parse_tests); i++) { + struct test_bitmap_parselist test = parse_tests[i]; + size_t len = test.flags & NO_LEN ? UINT_MAX : strlen(test.in); + + time = ktime_get(); + err = bitmap_parse(test.in, len, bmap, test.nbits); + time = ktime_get() - time; + + if (err != test.errno) { + pr_err("parse: %d: input is %s, errno is %d, expected %d\n", + i, test.in, err, test.errno); + continue; + } + + if (!err && test.expected + && !__bitmap_equal(bmap, test.expected, test.nbits)) { + pr_err("parse: %d: input is %s, result is 0x%lx, expected 0x%lx\n", + i, test.in, bmap[0], + *test.expected); + continue; + } + + if (test.flags & PARSE_TIME) + pr_err("parse: %d: input is '%s' OK, Time: %llu\n", + i, test.in, time); + } } #define EXP1_IN_BITS (sizeof(exp1) * 8) @@ -508,6 +619,196 @@ static void __init test_for_each_set_clump8(void) expect_eq_clump8(start, CLUMP_EXP_NUMBITS, clump_exp, &clump); } +struct test_bitmap_cut { + unsigned int first; + unsigned int cut; + unsigned int nbits; + unsigned long in[4]; + unsigned long expected[4]; +}; + +static struct test_bitmap_cut test_cut[] = { + { 0, 0, 8, { 0x0000000aUL, }, { 0x0000000aUL, }, }, + { 0, 0, 32, { 0xdadadeadUL, }, { 0xdadadeadUL, }, }, + { 0, 3, 8, { 0x000000aaUL, }, { 0x00000015UL, }, }, + { 3, 3, 8, { 0x000000aaUL, }, { 0x00000012UL, }, }, + { 0, 1, 32, { 0xa5a5a5a5UL, }, { 0x52d2d2d2UL, }, }, + { 0, 8, 32, { 0xdeadc0deUL, }, { 0x00deadc0UL, }, }, + { 1, 1, 32, { 0x5a5a5a5aUL, }, { 0x2d2d2d2cUL, }, }, + { 0, 15, 32, { 0xa5a5a5a5UL, }, { 0x00014b4bUL, }, }, + { 0, 16, 32, { 0xa5a5a5a5UL, }, { 0x0000a5a5UL, }, }, + { 15, 15, 32, { 0xa5a5a5a5UL, }, { 0x000125a5UL, }, }, + { 15, 16, 32, { 0xa5a5a5a5UL, }, { 0x0000a5a5UL, }, }, + { 16, 15, 32, { 0xa5a5a5a5UL, }, { 0x0001a5a5UL, }, }, + + { BITS_PER_LONG, BITS_PER_LONG, BITS_PER_LONG, + { 0xa5a5a5a5UL, 0xa5a5a5a5UL, }, + { 0xa5a5a5a5UL, 0xa5a5a5a5UL, }, + }, + { 1, BITS_PER_LONG - 1, BITS_PER_LONG, + { 0xa5a5a5a5UL, 0xa5a5a5a5UL, }, + { 0x00000001UL, 0x00000001UL, }, + }, + + { 0, BITS_PER_LONG * 2, BITS_PER_LONG * 2 + 1, + { 0xa5a5a5a5UL, 0x00000001UL, 0x00000001UL, 0x00000001UL }, + { 0x00000001UL, }, + }, + { 16, BITS_PER_LONG * 2 + 1, BITS_PER_LONG * 2 + 1 + 16, + { 0x0000ffffUL, 0x5a5a5a5aUL, 0x5a5a5a5aUL, 0x5a5a5a5aUL }, + { 0x2d2dffffUL, }, + }, +}; + +static void __init test_bitmap_cut(void) +{ + unsigned long b[5], *in = &b[1], *out = &b[0]; /* Partial overlap */ + int i; + + for (i = 0; i < ARRAY_SIZE(test_cut); i++) { + struct test_bitmap_cut *t = &test_cut[i]; + + memcpy(in, t->in, sizeof(t->in)); + + bitmap_cut(out, in, t->first, t->cut, t->nbits); + + expect_eq_bitmap(t->expected, out, t->nbits); + } +} + +struct test_bitmap_print { + const unsigned long *bitmap; + unsigned long nbits; + const char *mask; + const char *list; +}; + +static const unsigned long small_bitmap[] __initconst = { + BITMAP_FROM_U64(0x3333333311111111ULL), +}; + +static const char small_mask[] __initconst = "33333333,11111111\n"; +static const char small_list[] __initconst = "0,4,8,12,16,20,24,28,32-33,36-37,40-41,44-45,48-49,52-53,56-57,60-61\n"; + +static const unsigned long large_bitmap[] __initconst = { + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), +}; + +static const char large_mask[] __initconst = "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111\n"; + +static const char large_list[] __initconst = /* more than 4KB */ + "0,4,8,12,16,20,24,28,32-33,36-37,40-41,44-45,48-49,52-53,56-57,60-61,64,68,72,76,80,84,88,92,96-97,100-101,104-1" + "05,108-109,112-113,116-117,120-121,124-125,128,132,136,140,144,148,152,156,160-161,164-165,168-169,172-173,176-1" + "77,180-181,184-185,188-189,192,196,200,204,208,212,216,220,224-225,228-229,232-233,236-237,240-241,244-245,248-2" + "49,252-253,256,260,264,268,272,276,280,284,288-289,292-293,296-297,300-301,304-305,308-309,312-313,316-317,320,3" + "24,328,332,336,340,344,348,352-353,356-357,360-361,364-365,368-369,372-373,376-377,380-381,384,388,392,396,400,4" + "04,408,412,416-417,420-421,424-425,428-429,432-433,436-437,440-441,444-445,448,452,456,460,464,468,472,476,480-4" + "81,484-485,488-489,492-493,496-497,500-501,504-505,508-509,512,516,520,524,528,532,536,540,544-545,548-549,552-5" + "53,556-557,560-561,564-565,568-569,572-573,576,580,584,588,592,596,600,604,608-609,612-613,616-617,620-621,624-6" + "25,628-629,632-633,636-637,640,644,648,652,656,660,664,668,672-673,676-677,680-681,684-685,688-689,692-693,696-6" + "97,700-701,704,708,712,716,720,724,728,732,736-737,740-741,744-745,748-749,752-753,756-757,760-761,764-765,768,7" + "72,776,780,784,788,792,796,800-801,804-805,808-809,812-813,816-817,820-821,824-825,828-829,832,836,840,844,848,8" + "52,856,860,864-865,868-869,872-873,876-877,880-881,884-885,888-889,892-893,896,900,904,908,912,916,920,924,928-9" + "29,932-933,936-937,940-941,944-945,948-949,952-953,956-957,960,964,968,972,976,980,984,988,992-993,996-997,1000-" + "1001,1004-1005,1008-1009,1012-1013,1016-1017,1020-1021,1024,1028,1032,1036,1040,1044,1048,1052,1056-1057,1060-10" + "61,1064-1065,1068-1069,1072-1073,1076-1077,1080-1081,1084-1085,1088,1092,1096,1100,1104,1108,1112,1116,1120-1121" + ",1124-1125,1128-1129,1132-1133,1136-1137,1140-1141,1144-1145,1148-1149,1152,1156,1160,1164,1168,1172,1176,1180,1" + "184-1185,1188-1189,1192-1193,1196-1197,1200-1201,1204-1205,1208-1209,1212-1213,1216,1220,1224,1228,1232,1236,124" + "0,1244,1248-1249,1252-1253,1256-1257,1260-1261,1264-1265,1268-1269,1272-1273,1276-1277,1280,1284,1288,1292,1296," + "1300,1304,1308,1312-1313,1316-1317,1320-1321,1324-1325,1328-1329,1332-1333,1336-1337,1340-1341,1344,1348,1352,13" + "56,1360,1364,1368,1372,1376-1377,1380-1381,1384-1385,1388-1389,1392-1393,1396-1397,1400-1401,1404-1405,1408,1412" + ",1416,1420,1424,1428,1432,1436,1440-1441,1444-1445,1448-1449,1452-1453,1456-1457,1460-1461,1464-1465,1468-1469,1" + "472,1476,1480,1484,1488,1492,1496,1500,1504-1505,1508-1509,1512-1513,1516-1517,1520-1521,1524-1525,1528-1529,153" + "2-1533,1536,1540,1544,1548,1552,1556,1560,1564,1568-1569,1572-1573,1576-1577,1580-1581,1584-1585,1588-1589,1592-" + "1593,1596-1597,1600,1604,1608,1612,1616,1620,1624,1628,1632-1633,1636-1637,1640-1641,1644-1645,1648-1649,1652-16" + "53,1656-1657,1660-1661,1664,1668,1672,1676,1680,1684,1688,1692,1696-1697,1700-1701,1704-1705,1708-1709,1712-1713" + ",1716-1717,1720-1721,1724-1725,1728,1732,1736,1740,1744,1748,1752,1756,1760-1761,1764-1765,1768-1769,1772-1773,1" + "776-1777,1780-1781,1784-1785,1788-1789,1792,1796,1800,1804,1808,1812,1816,1820,1824-1825,1828-1829,1832-1833,183" + "6-1837,1840-1841,1844-1845,1848-1849,1852-1853,1856,1860,1864,1868,1872,1876,1880,1884,1888-1889,1892-1893,1896-" + "1897,1900-1901,1904-1905,1908-1909,1912-1913,1916-1917,1920,1924,1928,1932,1936,1940,1944,1948,1952-1953,1956-19" + "57,1960-1961,1964-1965,1968-1969,1972-1973,1976-1977,1980-1981,1984,1988,1992,1996,2000,2004,2008,2012,2016-2017" + ",2020-2021,2024-2025,2028-2029,2032-2033,2036-2037,2040-2041,2044-2045,2048,2052,2056,2060,2064,2068,2072,2076,2" + "080-2081,2084-2085,2088-2089,2092-2093,2096-2097,2100-2101,2104-2105,2108-2109,2112,2116,2120,2124,2128,2132,213" + "6,2140,2144-2145,2148-2149,2152-2153,2156-2157,2160-2161,2164-2165,2168-2169,2172-2173,2176,2180,2184,2188,2192," + "2196,2200,2204,2208-2209,2212-2213,2216-2217,2220-2221,2224-2225,2228-2229,2232-2233,2236-2237,2240,2244,2248,22" + "52,2256,2260,2264,2268,2272-2273,2276-2277,2280-2281,2284-2285,2288-2289,2292-2293,2296-2297,2300-2301,2304,2308" + ",2312,2316,2320,2324,2328,2332,2336-2337,2340-2341,2344-2345,2348-2349,2352-2353,2356-2357,2360-2361,2364-2365,2" + "368,2372,2376,2380,2384,2388,2392,2396,2400-2401,2404-2405,2408-2409,2412-2413,2416-2417,2420-2421,2424-2425,242" + "8-2429,2432,2436,2440,2444,2448,2452,2456,2460,2464-2465,2468-2469,2472-2473,2476-2477,2480-2481,2484-2485,2488-" + "2489,2492-2493,2496,2500,2504,2508,2512,2516,2520,2524,2528-2529,2532-2533,2536-2537,2540-2541,2544-2545,2548-25" + "49,2552-2553,2556-2557\n"; + +static const struct test_bitmap_print test_print[] __initconst = { + { small_bitmap, sizeof(small_bitmap) * BITS_PER_BYTE, small_mask, small_list }, + { large_bitmap, sizeof(large_bitmap) * BITS_PER_BYTE, large_mask, large_list }, +}; + +static void __init test_bitmap_print_buf(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(test_print); i++) { + const struct test_bitmap_print *t = &test_print[i]; + int n; + + n = bitmap_print_bitmask_to_buf(print_buf, t->bitmap, t->nbits, + 0, 2 * PAGE_SIZE); + expect_eq_uint(strlen(t->mask) + 1, n); + expect_eq_str(t->mask, print_buf, n); + + n = bitmap_print_list_to_buf(print_buf, t->bitmap, t->nbits, + 0, 2 * PAGE_SIZE); + expect_eq_uint(strlen(t->list) + 1, n); + expect_eq_str(t->list, print_buf, n); + + /* test by non-zero offset */ + if (strlen(t->list) > PAGE_SIZE) { + n = bitmap_print_list_to_buf(print_buf, t->bitmap, t->nbits, + PAGE_SIZE, PAGE_SIZE); + expect_eq_uint(strlen(t->list) + 1 - PAGE_SIZE, n); + expect_eq_str(t->list + PAGE_SIZE, print_buf, n); + } + } +} + static void __init selftest(void) { test_zero_clear(); @@ -515,10 +816,12 @@ static void __init selftest(void) test_copy(); test_replace(); test_bitmap_arr32(); + test_bitmap_parse(); test_bitmap_parselist(); - test_bitmap_parselist_user(); test_mem_optimisations(); test_for_each_set_clump8(); + test_bitmap_cut(); + test_bitmap_print_buf(); } KSTM_MODULE_LOADERS(test_bitmap); diff --git a/lib/test_bitops.c b/lib/test_bitops.c new file mode 100644 index 000000000000..3b7bcbee84db --- /dev/null +++ b/lib/test_bitops.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Intel Corporation + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/printk.h> + +/* a tiny module only meant to test + * + * set/clear_bit + * get_count_order/long + */ + +/* use an enum because that's the most common BITMAP usage */ +enum bitops_fun { + BITOPS_4 = 4, + BITOPS_7 = 7, + BITOPS_11 = 11, + BITOPS_31 = 31, + BITOPS_88 = 88, + BITOPS_LAST = 255, + BITOPS_LENGTH = 256 +}; + +static DECLARE_BITMAP(g_bitmap, BITOPS_LENGTH); + +static unsigned int order_comb[][2] = { + {0x00000003, 2}, + {0x00000004, 2}, + {0x00001fff, 13}, + {0x00002000, 13}, + {0x50000000, 31}, + {0x80000000, 31}, + {0x80003000, 32}, +}; + +#ifdef CONFIG_64BIT +static unsigned long order_comb_long[][2] = { + {0x0000000300000000, 34}, + {0x0000000400000000, 34}, + {0x00001fff00000000, 45}, + {0x0000200000000000, 45}, + {0x5000000000000000, 63}, + {0x8000000000000000, 63}, + {0x8000300000000000, 64}, +}; +#endif + +static int __init test_bitops_startup(void) +{ + int i, bit_set; + + pr_info("Starting bitops test\n"); + set_bit(BITOPS_4, g_bitmap); + set_bit(BITOPS_7, g_bitmap); + set_bit(BITOPS_11, g_bitmap); + set_bit(BITOPS_31, g_bitmap); + set_bit(BITOPS_88, g_bitmap); + + for (i = 0; i < ARRAY_SIZE(order_comb); i++) { + if (order_comb[i][1] != get_count_order(order_comb[i][0])) + pr_warn("get_count_order wrong for %x\n", + order_comb[i][0]); + } + + for (i = 0; i < ARRAY_SIZE(order_comb); i++) { + if (order_comb[i][1] != get_count_order_long(order_comb[i][0])) + pr_warn("get_count_order_long wrong for %x\n", + order_comb[i][0]); + } + +#ifdef CONFIG_64BIT + for (i = 0; i < ARRAY_SIZE(order_comb_long); i++) { + if (order_comb_long[i][1] != + get_count_order_long(order_comb_long[i][0])) + pr_warn("get_count_order_long wrong for %lx\n", + order_comb_long[i][0]); + } +#endif + + barrier(); + + clear_bit(BITOPS_4, g_bitmap); + clear_bit(BITOPS_7, g_bitmap); + clear_bit(BITOPS_11, g_bitmap); + clear_bit(BITOPS_31, g_bitmap); + clear_bit(BITOPS_88, g_bitmap); + + bit_set = find_first_bit(g_bitmap, BITOPS_LAST); + if (bit_set != BITOPS_LAST) + pr_err("ERROR: FOUND SET BIT %d\n", bit_set); + + pr_info("Completed bitops test\n"); + + return 0; +} + +static void __exit test_bitops_unstartup(void) +{ +} + +module_init(test_bitops_startup); +module_exit(test_bitops_unstartup); + +MODULE_AUTHOR("Jesse Brandeburg <jesse.brandeburg@intel.com>, Wei Yang <richard.weiyang@gmail.com>"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Bit testing module"); diff --git a/lib/test_bits.c b/lib/test_bits.c new file mode 100644 index 000000000000..c9368a2314e7 --- /dev/null +++ b/lib/test_bits.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Test cases for functions and macros in bits.h + */ + +#include <kunit/test.h> +#include <linux/bits.h> + + +static void genmask_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, 1ul, GENMASK(0, 0)); + KUNIT_EXPECT_EQ(test, 3ul, GENMASK(1, 0)); + KUNIT_EXPECT_EQ(test, 6ul, GENMASK(2, 1)); + KUNIT_EXPECT_EQ(test, 0xFFFFFFFFul, GENMASK(31, 0)); + +#ifdef TEST_GENMASK_FAILURES + /* these should fail compilation */ + GENMASK(0, 1); + GENMASK(0, 10); + GENMASK(9, 10); +#endif + + +} + +static void genmask_ull_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, 1ull, GENMASK_ULL(0, 0)); + KUNIT_EXPECT_EQ(test, 3ull, GENMASK_ULL(1, 0)); + KUNIT_EXPECT_EQ(test, 0x000000ffffe00000ull, GENMASK_ULL(39, 21)); + KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_ULL(63, 0)); + +#ifdef TEST_GENMASK_FAILURES + /* these should fail compilation */ + GENMASK_ULL(0, 1); + GENMASK_ULL(0, 10); + GENMASK_ULL(9, 10); +#endif +} + +static void genmask_input_check_test(struct kunit *test) +{ + unsigned int x, y; + int z, w; + + /* Unknown input */ + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(x, 0)); + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(0, x)); + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(x, y)); + + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(z, 0)); + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(0, z)); + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(z, w)); + + /* Valid input */ + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(1, 1)); + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(39, 21)); +} + + +static struct kunit_case bits_test_cases[] = { + KUNIT_CASE(genmask_test), + KUNIT_CASE(genmask_ull_test), + KUNIT_CASE(genmask_input_check_test), + {} +}; + +static struct kunit_suite bits_test_suite = { + .name = "bits-test", + .test_cases = bits_test_cases, +}; +kunit_test_suite(bits_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/lib/test_bpf.c b/lib/test_bpf.c index cecb230833be..adae39567264 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -52,6 +52,7 @@ #define FLAG_NO_DATA BIT(0) #define FLAG_EXPECTED_FAIL BIT(1) #define FLAG_SKB_FRAG BIT(2) +#define FLAG_VERIFIER_ZEXT BIT(3) enum { CLASSIC = BIT(6), /* Old BPF instructions only. */ @@ -80,6 +81,7 @@ struct bpf_test { int expected_errcode; /* used when FLAG_EXPECTED_FAIL is set in the aux */ __u8 frag_data[MAX_DATA]; int stack_depth; /* for eBPF only, since tests don't call verifier */ + int nr_testruns; /* Custom run count, defaults to MAX_TESTRUNS if 0 */ }; /* Large test cases need separate allocation and fill handler. */ @@ -345,7 +347,7 @@ static int __bpf_fill_ja(struct bpf_test *self, unsigned int len, static int bpf_fill_maxinsns11(struct bpf_test *self) { - /* Hits 70 passes on x86_64, so cannot get JITed there. */ + /* Hits 70 passes on x86_64 and triggers NOPs padding. */ return __bpf_fill_ja(self, BPF_MAXINSNS, 68); } @@ -461,6 +463,2602 @@ static int bpf_fill_stxdw(struct bpf_test *self) return __bpf_fill_stxdw(self, BPF_DW); } +static int __bpf_ld_imm64(struct bpf_insn insns[2], u8 reg, s64 imm64) +{ + struct bpf_insn tmp[] = {BPF_LD_IMM64(reg, imm64)}; + + memcpy(insns, tmp, sizeof(tmp)); + return 2; +} + +/* + * Branch conversion tests. Complex operations can expand to a lot + * of instructions when JITed. This in turn may cause jump offsets + * to overflow the field size of the native instruction, triggering + * a branch conversion mechanism in some JITs. + */ +static int __bpf_fill_max_jmp(struct bpf_test *self, int jmp, int imm) +{ + struct bpf_insn *insns; + int len = S16_MAX + 5; + int i; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + i = __bpf_ld_imm64(insns, R1, 0x0123456789abcdefULL); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insns[i++] = BPF_JMP_IMM(jmp, R0, imm, S16_MAX); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 2); + insns[i++] = BPF_EXIT_INSN(); + + while (i < len - 1) { + static const int ops[] = { + BPF_LSH, BPF_RSH, BPF_ARSH, BPF_ADD, + BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, + }; + int op = ops[(i >> 1) % ARRAY_SIZE(ops)]; + + if (i & 1) + insns[i++] = BPF_ALU32_REG(op, R0, R1); + else + insns[i++] = BPF_ALU64_REG(op, R0, R1); + } + + insns[i++] = BPF_EXIT_INSN(); + self->u.ptr.insns = insns; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +/* Branch taken by runtime decision */ +static int bpf_fill_max_jmp_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JEQ, 1); +} + +/* Branch not taken by runtime decision */ +static int bpf_fill_max_jmp_not_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JEQ, 0); +} + +/* Branch always taken, known at JIT time */ +static int bpf_fill_max_jmp_always_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JGE, 0); +} + +/* Branch never taken, known at JIT time */ +static int bpf_fill_max_jmp_never_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JLT, 0); +} + +/* ALU result computation used in tests */ +static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) +{ + *res = 0; + switch (op) { + case BPF_MOV: + *res = v2; + break; + case BPF_AND: + *res = v1 & v2; + break; + case BPF_OR: + *res = v1 | v2; + break; + case BPF_XOR: + *res = v1 ^ v2; + break; + case BPF_LSH: + *res = v1 << v2; + break; + case BPF_RSH: + *res = v1 >> v2; + break; + case BPF_ARSH: + *res = v1 >> v2; + if (v2 > 0 && v1 > S64_MAX) + *res |= ~0ULL << (64 - v2); + break; + case BPF_ADD: + *res = v1 + v2; + break; + case BPF_SUB: + *res = v1 - v2; + break; + case BPF_MUL: + *res = v1 * v2; + break; + case BPF_DIV: + if (v2 == 0) + return false; + *res = div64_u64(v1, v2); + break; + case BPF_MOD: + if (v2 == 0) + return false; + div64_u64_rem(v1, v2, res); + break; + } + return true; +} + +/* Test an ALU shift operation for all valid shift values */ +static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, + u8 mode, bool alu32) +{ + static const s64 regs[] = { + 0x0123456789abcdefLL, /* dword > 0, word < 0 */ + 0xfedcba9876543210LL, /* dowrd < 0, word > 0 */ + 0xfedcba0198765432LL, /* dowrd < 0, word < 0 */ + 0x0123458967abcdefLL, /* dword > 0, word > 0 */ + }; + int bits = alu32 ? 32 : 64; + int len = (2 + 7 * bits) * ARRAY_SIZE(regs) + 3; + struct bpf_insn *insn; + int imm, k; + int i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (k = 0; k < ARRAY_SIZE(regs); k++) { + s64 reg = regs[k]; + + i += __bpf_ld_imm64(&insn[i], R3, reg); + + for (imm = 0; imm < bits; imm++) { + u64 val; + + /* Perform operation */ + insn[i++] = BPF_ALU64_REG(BPF_MOV, R1, R3); + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R2, imm); + if (alu32) { + if (mode == BPF_K) + insn[i++] = BPF_ALU32_IMM(op, R1, imm); + else + insn[i++] = BPF_ALU32_REG(op, R1, R2); + + if (op == BPF_ARSH) + reg = (s32)reg; + else + reg = (u32)reg; + __bpf_alu_result(&val, reg, imm, op); + val = (u32)val; + } else { + if (mode == BPF_K) + insn[i++] = BPF_ALU64_IMM(op, R1, imm); + else + insn[i++] = BPF_ALU64_REG(op, R1, R2); + __bpf_alu_result(&val, reg, imm, op); + } + + /* + * When debugging a JIT that fails this test, one + * can write the immediate value to R0 here to find + * out which operand values that fail. + */ + + /* Load reference and check the result */ + i += __bpf_ld_imm64(&insn[i], R4, val); + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R4, 1); + insn[i++] = BPF_EXIT_INSN(); + } + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +static int bpf_fill_alu64_lsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, false); +} + +static int bpf_fill_alu64_rsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, false); +} + +static int bpf_fill_alu64_arsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, false); +} + +static int bpf_fill_alu64_lsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, false); +} + +static int bpf_fill_alu64_rsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, false); +} + +static int bpf_fill_alu64_arsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, false); +} + +static int bpf_fill_alu32_lsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, true); +} + +static int bpf_fill_alu32_rsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, true); +} + +static int bpf_fill_alu32_arsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, true); +} + +static int bpf_fill_alu32_lsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, true); +} + +static int bpf_fill_alu32_rsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, true); +} + +static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, true); +} + +/* + * Test an ALU register shift operation for all valid shift values + * for the case when the source and destination are the same. + */ +static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, + bool alu32) +{ + int bits = alu32 ? 32 : 64; + int len = 3 + 6 * bits; + struct bpf_insn *insn; + int i = 0; + u64 val; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (val = 0; val < bits; val++) { + u64 res; + + /* Perform operation */ + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R1, val); + if (alu32) + insn[i++] = BPF_ALU32_REG(op, R1, R1); + else + insn[i++] = BPF_ALU64_REG(op, R1, R1); + + /* Compute the reference result */ + __bpf_alu_result(&res, val, val, op); + if (alu32) + res = (u32)res; + i += __bpf_ld_imm64(&insn[i], R2, res); + + /* Check the actual result */ + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); + insn[i++] = BPF_EXIT_INSN(); + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +static int bpf_fill_alu64_lsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, false); +} + +static int bpf_fill_alu64_rsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, false); +} + +static int bpf_fill_alu64_arsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, false); +} + +static int bpf_fill_alu32_lsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, true); +} + +static int bpf_fill_alu32_rsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, true); +} + +static int bpf_fill_alu32_arsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, true); +} + +/* + * Common operand pattern generator for exhaustive power-of-two magnitudes + * tests. The block size parameters can be adjusted to increase/reduce the + * number of combinatons tested and thereby execution speed and memory + * footprint. + */ + +static inline s64 value(int msb, int delta, int sign) +{ + return sign * (1LL << msb) + delta; +} + +static int __bpf_fill_pattern(struct bpf_test *self, void *arg, + int dbits, int sbits, int block1, int block2, + int (*emit)(struct bpf_test*, void*, + struct bpf_insn*, s64, s64)) +{ + static const int sgn[][2] = {{1, 1}, {1, -1}, {-1, 1}, {-1, -1}}; + struct bpf_insn *insns; + int di, si, bt, db, sb; + int count, len, k; + int extra = 1 + 2; + int i = 0; + + /* Total number of iterations for the two pattern */ + count = (dbits - 1) * (sbits - 1) * block1 * block1 * ARRAY_SIZE(sgn); + count += (max(dbits, sbits) - 1) * block2 * block2 * ARRAY_SIZE(sgn); + + /* Compute the maximum number of insns and allocate the buffer */ + len = extra + count * (*emit)(self, arg, NULL, 0, 0); + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Add head instruction(s) */ + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + /* + * Pattern 1: all combinations of power-of-two magnitudes and sign, + * and with a block of contiguous values around each magnitude. + */ + for (di = 0; di < dbits - 1; di++) /* Dst magnitudes */ + for (si = 0; si < sbits - 1; si++) /* Src magnitudes */ + for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ + for (db = -(block1 / 2); + db < (block1 + 1) / 2; db++) + for (sb = -(block1 / 2); + sb < (block1 + 1) / 2; sb++) { + s64 dst, src; + + dst = value(di, db, sgn[k][0]); + src = value(si, sb, sgn[k][1]); + i += (*emit)(self, arg, + &insns[i], + dst, src); + } + /* + * Pattern 2: all combinations for a larger block of values + * for each power-of-two magnitude and sign, where the magnitude is + * the same for both operands. + */ + for (bt = 0; bt < max(dbits, sbits) - 1; bt++) /* Magnitude */ + for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ + for (db = -(block2 / 2); db < (block2 + 1) / 2; db++) + for (sb = -(block2 / 2); + sb < (block2 + 1) / 2; sb++) { + s64 dst, src; + + dst = value(bt % dbits, db, sgn[k][0]); + src = value(bt % sbits, sb, sgn[k][1]); + i += (*emit)(self, arg, &insns[i], + dst, src); + } + + /* Append tail instructions */ + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insns[i++] = BPF_EXIT_INSN(); + BUG_ON(i > len); + + self->u.ptr.insns = insns; + self->u.ptr.len = i; + + return 0; +} + +/* + * Block size parameters used in pattern tests below. une as needed to + * increase/reduce the number combinations tested, see following examples. + * block values per operand MSB + * ---------------------------------------- + * 0 none + * 1 (1 << MSB) + * 2 (1 << MSB) + [-1, 0] + * 3 (1 << MSB) + [-1, 0, 1] + */ +#define PATTERN_BLOCK1 1 +#define PATTERN_BLOCK2 5 + +/* Number of test runs for a pattern test */ +#define NR_PATTERN_RUNS 1 + +/* + * Exhaustive tests of ALU operations for all combinations of power-of-two + * magnitudes of the operands, both for positive and negative values. The + * test is designed to verify e.g. the ALU and ALU64 operations for JITs that + * emit different code depending on the magnitude of the immediate value. + */ +static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 7; + + if (__bpf_alu_result(&res, dst, (s32)imm, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R3, res); + insns[i++] = BPF_ALU64_IMM(op, R1, imm); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_emit_alu32_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 7; + + if (__bpf_alu_result(&res, (u32)dst, (u32)imm, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R3, (u32)res); + insns[i++] = BPF_ALU32_IMM(op, R1, imm); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_emit_alu64_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 9; + + if (__bpf_alu_result(&res, dst, src, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, res); + insns[i++] = BPF_ALU64_REG(op, R1, R2); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 9; + + if (__bpf_alu_result(&res, (u32)dst, (u32)src, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, (u32)res); + insns[i++] = BPF_ALU32_REG(op, R1, R2); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_fill_alu64_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu64_imm); +} + +static int __bpf_fill_alu32_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu32_imm); +} + +static int __bpf_fill_alu64_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu64_reg); +} + +static int __bpf_fill_alu32_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu32_reg); +} + +/* ALU64 immediate operations */ +static int bpf_fill_alu64_mov_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MOV); +} + +static int bpf_fill_alu64_and_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_AND); +} + +static int bpf_fill_alu64_or_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_OR); +} + +static int bpf_fill_alu64_xor_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_XOR); +} + +static int bpf_fill_alu64_add_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_ADD); +} + +static int bpf_fill_alu64_sub_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_SUB); +} + +static int bpf_fill_alu64_mul_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MUL); +} + +static int bpf_fill_alu64_div_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_DIV); +} + +static int bpf_fill_alu64_mod_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MOD); +} + +/* ALU32 immediate operations */ +static int bpf_fill_alu32_mov_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MOV); +} + +static int bpf_fill_alu32_and_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_AND); +} + +static int bpf_fill_alu32_or_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_OR); +} + +static int bpf_fill_alu32_xor_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_XOR); +} + +static int bpf_fill_alu32_add_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_ADD); +} + +static int bpf_fill_alu32_sub_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_SUB); +} + +static int bpf_fill_alu32_mul_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MUL); +} + +static int bpf_fill_alu32_div_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_DIV); +} + +static int bpf_fill_alu32_mod_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MOD); +} + +/* ALU64 register operations */ +static int bpf_fill_alu64_mov_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_MOV); +} + +static int bpf_fill_alu64_and_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_AND); +} + +static int bpf_fill_alu64_or_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_OR); +} + +static int bpf_fill_alu64_xor_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_XOR); +} + +static int bpf_fill_alu64_add_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_ADD); +} + +static int bpf_fill_alu64_sub_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_SUB); +} + +static int bpf_fill_alu64_mul_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_MUL); +} + +static int bpf_fill_alu64_div_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_DIV); +} + +static int bpf_fill_alu64_mod_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_MOD); +} + +/* ALU32 register operations */ +static int bpf_fill_alu32_mov_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_MOV); +} + +static int bpf_fill_alu32_and_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_AND); +} + +static int bpf_fill_alu32_or_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_OR); +} + +static int bpf_fill_alu32_xor_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_XOR); +} + +static int bpf_fill_alu32_add_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_ADD); +} + +static int bpf_fill_alu32_sub_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_SUB); +} + +static int bpf_fill_alu32_mul_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_MUL); +} + +static int bpf_fill_alu32_div_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_DIV); +} + +static int bpf_fill_alu32_mod_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_MOD); +} + +/* + * Test JITs that implement complex ALU operations as function + * calls, and must re-arrange operands for argument passing. + */ +static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32) +{ + int len = 2 + 10 * 10; + struct bpf_insn *insns; + u64 dst, res; + int i = 0; + u32 imm; + int rd; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Operand and result values according to operation */ + if (alu32) + dst = 0x76543210U; + else + dst = 0x7edcba9876543210ULL; + imm = 0x01234567U; + + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + imm &= 31; + + __bpf_alu_result(&res, dst, imm, op); + + if (alu32) + res = (u32)res; + + /* Check all operand registers */ + for (rd = R0; rd <= R9; rd++) { + i += __bpf_ld_imm64(&insns[i], rd, dst); + + if (alu32) + insns[i++] = BPF_ALU32_IMM(op, rd, imm); + else + insns[i++] = BPF_ALU64_IMM(op, rd, imm); + + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res >> 32, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + } + + insns[i++] = BPF_MOV64_IMM(R0, 1); + insns[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insns; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +/* ALU64 K registers */ +static int bpf_fill_alu64_mov_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOV, false); +} + +static int bpf_fill_alu64_and_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_AND, false); +} + +static int bpf_fill_alu64_or_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_OR, false); +} + +static int bpf_fill_alu64_xor_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_XOR, false); +} + +static int bpf_fill_alu64_lsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_LSH, false); +} + +static int bpf_fill_alu64_rsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_RSH, false); +} + +static int bpf_fill_alu64_arsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ARSH, false); +} + +static int bpf_fill_alu64_add_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ADD, false); +} + +static int bpf_fill_alu64_sub_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_SUB, false); +} + +static int bpf_fill_alu64_mul_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MUL, false); +} + +static int bpf_fill_alu64_div_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_DIV, false); +} + +static int bpf_fill_alu64_mod_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOD, false); +} + +/* ALU32 K registers */ +static int bpf_fill_alu32_mov_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOV, true); +} + +static int bpf_fill_alu32_and_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_AND, true); +} + +static int bpf_fill_alu32_or_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_OR, true); +} + +static int bpf_fill_alu32_xor_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_XOR, true); +} + +static int bpf_fill_alu32_lsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_LSH, true); +} + +static int bpf_fill_alu32_rsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_RSH, true); +} + +static int bpf_fill_alu32_arsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ARSH, true); +} + +static int bpf_fill_alu32_add_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ADD, true); +} + +static int bpf_fill_alu32_sub_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_SUB, true); +} + +static int bpf_fill_alu32_mul_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MUL, true); +} + +static int bpf_fill_alu32_div_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_DIV, true); +} + +static int bpf_fill_alu32_mod_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOD, true); +} + +/* + * Test JITs that implement complex ALU operations as function + * calls, and must re-arrange operands for argument passing. + */ +static int __bpf_fill_alu_reg_pairs(struct bpf_test *self, u8 op, bool alu32) +{ + int len = 2 + 10 * 10 * 12; + u64 dst, src, res, same; + struct bpf_insn *insns; + int rd, rs; + int i = 0; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Operand and result values according to operation */ + if (alu32) { + dst = 0x76543210U; + src = 0x01234567U; + } else { + dst = 0x7edcba9876543210ULL; + src = 0x0123456789abcdefULL; + } + + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + src &= 31; + + __bpf_alu_result(&res, dst, src, op); + __bpf_alu_result(&same, src, src, op); + + if (alu32) { + res = (u32)res; + same = (u32)same; + } + + /* Check all combinations of operand registers */ + for (rd = R0; rd <= R9; rd++) { + for (rs = R0; rs <= R9; rs++) { + u64 val = rd == rs ? same : res; + + i += __bpf_ld_imm64(&insns[i], rd, dst); + i += __bpf_ld_imm64(&insns[i], rs, src); + + if (alu32) + insns[i++] = BPF_ALU32_REG(op, rd, rs); + else + insns[i++] = BPF_ALU64_REG(op, rd, rs); + + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val >> 32, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + } + } + + insns[i++] = BPF_MOV64_IMM(R0, 1); + insns[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insns; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +/* ALU64 X register combinations */ +static int bpf_fill_alu64_mov_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOV, false); +} + +static int bpf_fill_alu64_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_AND, false); +} + +static int bpf_fill_alu64_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_OR, false); +} + +static int bpf_fill_alu64_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_XOR, false); +} + +static int bpf_fill_alu64_lsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_LSH, false); +} + +static int bpf_fill_alu64_rsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_RSH, false); +} + +static int bpf_fill_alu64_arsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, false); +} + +static int bpf_fill_alu64_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ADD, false); +} + +static int bpf_fill_alu64_sub_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_SUB, false); +} + +static int bpf_fill_alu64_mul_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MUL, false); +} + +static int bpf_fill_alu64_div_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_DIV, false); +} + +static int bpf_fill_alu64_mod_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOD, false); +} + +/* ALU32 X register combinations */ +static int bpf_fill_alu32_mov_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOV, true); +} + +static int bpf_fill_alu32_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_AND, true); +} + +static int bpf_fill_alu32_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_OR, true); +} + +static int bpf_fill_alu32_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_XOR, true); +} + +static int bpf_fill_alu32_lsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_LSH, true); +} + +static int bpf_fill_alu32_rsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_RSH, true); +} + +static int bpf_fill_alu32_arsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, true); +} + +static int bpf_fill_alu32_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ADD, true); +} + +static int bpf_fill_alu32_sub_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_SUB, true); +} + +static int bpf_fill_alu32_mul_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MUL, true); +} + +static int bpf_fill_alu32_div_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_DIV, true); +} + +static int bpf_fill_alu32_mod_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOD, true); +} + +/* + * Exhaustive tests of atomic operations for all power-of-two operand + * magnitudes, both for positive and negative values. + */ + +static int __bpf_emit_atomic64(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + u64 keep, fetch, res; + int i = 0; + + if (!insns) + return 21; + + switch (op) { + case BPF_XCHG: + res = src; + break; + default: + __bpf_alu_result(&res, dst, src, BPF_OP(op)); + } + + keep = 0x0123456789abcdefULL; + if (op & BPF_FETCH) + fetch = dst; + else + fetch = src; + + i += __bpf_ld_imm64(&insns[i], R0, keep); + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, res); + i += __bpf_ld_imm64(&insns[i], R4, fetch); + i += __bpf_ld_imm64(&insns[i], R5, keep); + + insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); + insns[i++] = BPF_ATOMIC_OP(BPF_DW, op, R10, R2, -8); + insns[i++] = BPF_LDX_MEM(BPF_DW, R1, R10, -8); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_emit_atomic32(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + u64 keep, fetch, res; + int i = 0; + + if (!insns) + return 21; + + switch (op) { + case BPF_XCHG: + res = src; + break; + default: + __bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op)); + } + + keep = 0x0123456789abcdefULL; + if (op & BPF_FETCH) + fetch = (u32)dst; + else + fetch = src; + + i += __bpf_ld_imm64(&insns[i], R0, keep); + i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, (u32)res); + i += __bpf_ld_imm64(&insns[i], R4, fetch); + i += __bpf_ld_imm64(&insns[i], R5, keep); + + insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); + insns[i++] = BPF_ATOMIC_OP(BPF_W, op, R10, R2, -4); + insns[i++] = BPF_LDX_MEM(BPF_W, R1, R10, -4); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_emit_cmpxchg64(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int i = 0; + + if (!insns) + return 23; + + i += __bpf_ld_imm64(&insns[i], R0, ~dst); + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + + /* Result unsuccessful */ + insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); + insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); + insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + /* Result successful */ + insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); + insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R3, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_emit_cmpxchg32(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int i = 0; + + if (!insns) + return 27; + + i += __bpf_ld_imm64(&insns[i], R0, ~dst); + i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + + /* Result unsuccessful */ + insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); + insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); + insns[i++] = BPF_ZEXT_REG(R0), /* Zext always inserted by verifier */ + insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); + + insns[i++] = BPF_JMP32_REG(BPF_JEQ, R1, R3, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + /* Result successful */ + i += __bpf_ld_imm64(&insns[i], R0, dst); + insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); + insns[i++] = BPF_ZEXT_REG(R0), /* Zext always inserted by verifier */ + insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); + + insns[i++] = BPF_JMP32_REG(BPF_JEQ, R2, R3, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_fill_atomic64(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + 0, PATTERN_BLOCK2, + &__bpf_emit_atomic64); +} + +static int __bpf_fill_atomic32(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + 0, PATTERN_BLOCK2, + &__bpf_emit_atomic32); +} + +/* 64-bit atomic operations */ +static int bpf_fill_atomic64_add(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_ADD); +} + +static int bpf_fill_atomic64_and(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_AND); +} + +static int bpf_fill_atomic64_or(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_OR); +} + +static int bpf_fill_atomic64_xor(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_XOR); +} + +static int bpf_fill_atomic64_add_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic64_and_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic64_or_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xor_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xchg(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_XCHG); +} + +static int bpf_fill_cmpxchg64(struct bpf_test *self) +{ + return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, + &__bpf_emit_cmpxchg64); +} + +/* 32-bit atomic operations */ +static int bpf_fill_atomic32_add(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_ADD); +} + +static int bpf_fill_atomic32_and(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_AND); +} + +static int bpf_fill_atomic32_or(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_OR); +} + +static int bpf_fill_atomic32_xor(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_XOR); +} + +static int bpf_fill_atomic32_add_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic32_and_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic32_or_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xor_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xchg(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_XCHG); +} + +static int bpf_fill_cmpxchg32(struct bpf_test *self) +{ + return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, + &__bpf_emit_cmpxchg32); +} + +/* + * Test JITs that implement ATOMIC operations as function calls or + * other primitives, and must re-arrange operands for argument passing. + */ +static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op) +{ + struct bpf_insn *insn; + int len = 2 + 34 * 10 * 10; + u64 mem, upd, res; + int rd, rs, i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + /* Operand and memory values */ + if (width == BPF_DW) { + mem = 0x0123456789abcdefULL; + upd = 0xfedcba9876543210ULL; + } else { /* BPF_W */ + mem = 0x01234567U; + upd = 0x76543210U; + } + + /* Memory updated according to operation */ + switch (op) { + case BPF_XCHG: + res = upd; + break; + case BPF_CMPXCHG: + res = mem; + break; + default: + __bpf_alu_result(&res, mem, upd, BPF_OP(op)); + } + + /* Test all operand registers */ + for (rd = R0; rd <= R9; rd++) { + for (rs = R0; rs <= R9; rs++) { + u64 cmp, src; + + /* Initialize value in memory */ + i += __bpf_ld_imm64(&insn[i], R0, mem); + insn[i++] = BPF_STX_MEM(width, R10, R0, -8); + + /* Initialize registers in order */ + i += __bpf_ld_imm64(&insn[i], R0, ~mem); + i += __bpf_ld_imm64(&insn[i], rs, upd); + insn[i++] = BPF_MOV64_REG(rd, R10); + + /* Perform atomic operation */ + insn[i++] = BPF_ATOMIC_OP(width, op, rd, rs, -8); + if (op == BPF_CMPXCHG && width == BPF_W) + insn[i++] = BPF_ZEXT_REG(R0); + + /* Check R0 register value */ + if (op == BPF_CMPXCHG) + cmp = mem; /* Expect value from memory */ + else if (R0 == rd || R0 == rs) + cmp = 0; /* Aliased, checked below */ + else + cmp = ~mem; /* Expect value to be preserved */ + if (cmp) { + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, + (u32)cmp, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + insn[i++] = BPF_ALU64_IMM(BPF_RSH, R0, 32); + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, + cmp >> 32, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + + /* Check source register value */ + if (rs == R0 && op == BPF_CMPXCHG) + src = 0; /* Aliased with R0, checked above */ + else if (rs == rd && (op == BPF_CMPXCHG || + !(op & BPF_FETCH))) + src = 0; /* Aliased with rd, checked below */ + else if (op == BPF_CMPXCHG) + src = upd; /* Expect value to be preserved */ + else if (op & BPF_FETCH) + src = mem; /* Expect fetched value from mem */ + else /* no fetch */ + src = upd; /* Expect value to be preserved */ + if (src) { + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, + (u32)src, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + insn[i++] = BPF_ALU64_IMM(BPF_RSH, rs, 32); + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, + src >> 32, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + + /* Check destination register value */ + if (!(rd == R0 && op == BPF_CMPXCHG) && + !(rd == rs && (op & BPF_FETCH))) { + insn[i++] = BPF_JMP_REG(BPF_JEQ, rd, R10, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + + /* Check value in memory */ + if (rs != rd) { /* No aliasing */ + i += __bpf_ld_imm64(&insn[i], R1, res); + } else if (op == BPF_XCHG) { /* Aliased, XCHG */ + insn[i++] = BPF_MOV64_REG(R1, R10); + } else if (op == BPF_CMPXCHG) { /* Aliased, CMPXCHG */ + i += __bpf_ld_imm64(&insn[i], R1, mem); + } else { /* Aliased, ALU oper */ + i += __bpf_ld_imm64(&insn[i], R1, mem); + insn[i++] = BPF_ALU64_REG(BPF_OP(op), R1, R10); + } + + insn[i++] = BPF_LDX_MEM(width, R0, R10, -8); + if (width == BPF_DW) + insn[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); + else /* width == BPF_W */ + insn[i++] = BPF_JMP32_REG(BPF_JEQ, R0, R1, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + } + + insn[i++] = BPF_MOV64_IMM(R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = i; + BUG_ON(i > len); + + return 0; +} + +/* 64-bit atomic register tests */ +static int bpf_fill_atomic64_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD); +} + +static int bpf_fill_atomic64_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND); +} + +static int bpf_fill_atomic64_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR); +} + +static int bpf_fill_atomic64_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR); +} + +static int bpf_fill_atomic64_add_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic64_and_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic64_or_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xor_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XCHG); +} + +static int bpf_fill_atomic64_cmpxchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_CMPXCHG); +} + +/* 32-bit atomic register tests */ +static int bpf_fill_atomic32_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD); +} + +static int bpf_fill_atomic32_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND); +} + +static int bpf_fill_atomic32_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR); +} + +static int bpf_fill_atomic32_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR); +} + +static int bpf_fill_atomic32_add_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic32_and_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic32_or_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xor_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XCHG); +} + +static int bpf_fill_atomic32_cmpxchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_CMPXCHG); +} + +/* + * Test the two-instruction 64-bit immediate load operation for all + * power-of-two magnitudes of the immediate operand. For each MSB, a block + * of immediate values centered around the power-of-two MSB are tested, + * both for positive and negative values. The test is designed to verify + * the operation for JITs that emit different code depending on the magnitude + * of the immediate value. This is often the case if the native instruction + * immediate field width is narrower than 32 bits. + */ +static int bpf_fill_ld_imm64_magn(struct bpf_test *self) +{ + int block = 64; /* Increase for more tests per MSB position */ + int len = 3 + 8 * 63 * block * 2; + struct bpf_insn *insn; + int bit, adj, sign; + int i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (bit = 0; bit <= 62; bit++) { + for (adj = -block / 2; adj < block / 2; adj++) { + for (sign = -1; sign <= 1; sign += 2) { + s64 imm = sign * ((1LL << bit) + adj); + + /* Perform operation */ + i += __bpf_ld_imm64(&insn[i], R1, imm); + + /* Load reference */ + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, + (u32)(imm >> 32)); + insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); + insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); + + /* Check result */ + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); + insn[i++] = BPF_EXIT_INSN(); + } + } + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +/* + * Test the two-instruction 64-bit immediate load operation for different + * combinations of bytes. Each byte in the 64-bit word is constructed as + * (base & mask) | (rand() & ~mask), where rand() is a deterministic LCG. + * All patterns (base1, mask1) and (base2, mask2) bytes are tested. + */ +static int __bpf_fill_ld_imm64_bytes(struct bpf_test *self, + u8 base1, u8 mask1, + u8 base2, u8 mask2) +{ + struct bpf_insn *insn; + int len = 3 + 8 * BIT(8); + int pattern, index; + u32 rand = 1; + int i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (pattern = 0; pattern < BIT(8); pattern++) { + u64 imm = 0; + + for (index = 0; index < 8; index++) { + int byte; + + if (pattern & BIT(index)) + byte = (base1 & mask1) | (rand & ~mask1); + else + byte = (base2 & mask2) | (rand & ~mask2); + imm = (imm << 8) | byte; + } + + /* Update our LCG */ + rand = rand * 1664525 + 1013904223; + + /* Perform operation */ + i += __bpf_ld_imm64(&insn[i], R1, imm); + + /* Load reference */ + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, (u32)(imm >> 32)); + insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); + insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); + + /* Check result */ + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); + insn[i++] = BPF_EXIT_INSN(); + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +static int bpf_fill_ld_imm64_checker(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 0, 0xff, 0xff, 0xff); +} + +static int bpf_fill_ld_imm64_pos_neg(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0x80, 0x80); +} + +static int bpf_fill_ld_imm64_pos_zero(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0, 0xff); +} + +static int bpf_fill_ld_imm64_neg_zero(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 0x80, 0x80, 0, 0xff); +} + +/* + * Exhaustive tests of JMP operations for all combinations of power-of-two + * magnitudes of the operands, both for positive and negative values. The + * test is designed to verify e.g. the JMP and JMP32 operations for JITs that + * emit different code depending on the magnitude of the immediate value. + */ + +static bool __bpf_match_jmp_cond(s64 v1, s64 v2, u8 op) +{ + switch (op) { + case BPF_JSET: + return !!(v1 & v2); + case BPF_JEQ: + return v1 == v2; + case BPF_JNE: + return v1 != v2; + case BPF_JGT: + return (u64)v1 > (u64)v2; + case BPF_JGE: + return (u64)v1 >= (u64)v2; + case BPF_JLT: + return (u64)v1 < (u64)v2; + case BPF_JLE: + return (u64)v1 <= (u64)v2; + case BPF_JSGT: + return v1 > v2; + case BPF_JSGE: + return v1 >= v2; + case BPF_JSLT: + return v1 < v2; + case BPF_JSLE: + return v1 <= v2; + } + return false; +} + +static int __bpf_emit_jmp_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond(dst, (s32)imm, op); + int i = 0; + + insns[i++] = BPF_ALU32_IMM(BPF_MOV, R0, match); + + i += __bpf_ld_imm64(&insns[i], R1, dst); + insns[i++] = BPF_JMP_IMM(op, R1, imm, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 5 + 1; +} + +static int __bpf_emit_jmp32_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond((s32)dst, (s32)imm, op); + int i = 0; + + i += __bpf_ld_imm64(&insns[i], R1, dst); + insns[i++] = BPF_JMP32_IMM(op, R1, imm, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 5; +} + +static int __bpf_emit_jmp_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond(dst, src, op); + int i = 0; + + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + insns[i++] = BPF_JMP_REG(op, R1, R2, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 7; +} + +static int __bpf_emit_jmp32_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond((s32)dst, (s32)src, op); + int i = 0; + + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + insns[i++] = BPF_JMP32_REG(op, R1, R2, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 7; +} + +static int __bpf_fill_jmp_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp_imm); +} + +static int __bpf_fill_jmp32_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp32_imm); +} + +static int __bpf_fill_jmp_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp_reg); +} + +static int __bpf_fill_jmp32_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp32_reg); +} + +/* JMP immediate tests */ +static int bpf_fill_jmp_jset_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSET); +} + +static int bpf_fill_jmp_jeq_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JEQ); +} + +static int bpf_fill_jmp_jne_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JNE); +} + +static int bpf_fill_jmp_jgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JGT); +} + +static int bpf_fill_jmp_jge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JGE); +} + +static int bpf_fill_jmp_jlt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JLT); +} + +static int bpf_fill_jmp_jle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JLE); +} + +static int bpf_fill_jmp_jsgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSGT); +} + +static int bpf_fill_jmp_jsge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSGE); +} + +static int bpf_fill_jmp_jslt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSLT); +} + +static int bpf_fill_jmp_jsle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSLE); +} + +/* JMP32 immediate tests */ +static int bpf_fill_jmp32_jset_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSET); +} + +static int bpf_fill_jmp32_jeq_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JEQ); +} + +static int bpf_fill_jmp32_jne_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JNE); +} + +static int bpf_fill_jmp32_jgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JGT); +} + +static int bpf_fill_jmp32_jge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JGE); +} + +static int bpf_fill_jmp32_jlt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JLT); +} + +static int bpf_fill_jmp32_jle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JLE); +} + +static int bpf_fill_jmp32_jsgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSGT); +} + +static int bpf_fill_jmp32_jsge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSGE); +} + +static int bpf_fill_jmp32_jslt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSLT); +} + +static int bpf_fill_jmp32_jsle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSLE); +} + +/* JMP register tests */ +static int bpf_fill_jmp_jset_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSET); +} + +static int bpf_fill_jmp_jeq_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JEQ); +} + +static int bpf_fill_jmp_jne_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JNE); +} + +static int bpf_fill_jmp_jgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JGT); +} + +static int bpf_fill_jmp_jge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JGE); +} + +static int bpf_fill_jmp_jlt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JLT); +} + +static int bpf_fill_jmp_jle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JLE); +} + +static int bpf_fill_jmp_jsgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSGT); +} + +static int bpf_fill_jmp_jsge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSGE); +} + +static int bpf_fill_jmp_jslt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSLT); +} + +static int bpf_fill_jmp_jsle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSLE); +} + +/* JMP32 register tests */ +static int bpf_fill_jmp32_jset_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSET); +} + +static int bpf_fill_jmp32_jeq_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JEQ); +} + +static int bpf_fill_jmp32_jne_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JNE); +} + +static int bpf_fill_jmp32_jgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JGT); +} + +static int bpf_fill_jmp32_jge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JGE); +} + +static int bpf_fill_jmp32_jlt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JLT); +} + +static int bpf_fill_jmp32_jle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JLE); +} + +static int bpf_fill_jmp32_jsgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSGT); +} + +static int bpf_fill_jmp32_jsge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSGE); +} + +static int bpf_fill_jmp32_jslt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSLT); +} + +static int bpf_fill_jmp32_jsle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSLE); +} + +/* + * Set up a sequence of staggered jumps, forwards and backwards with + * increasing offset. This tests the conversion of relative jumps to + * JITed native jumps. On some architectures, for example MIPS, a large + * PC-relative jump offset may overflow the immediate field of the native + * conditional branch instruction, triggering a conversion to use an + * absolute jump instead. Since this changes the jump offsets, another + * offset computation pass is necessary, and that may in turn trigger + * another branch conversion. This jump sequence is particularly nasty + * in that regard. + * + * The sequence generation is parameterized by size and jump type. + * The size must be even, and the expected result is always size + 1. + * Below is an example with size=8 and result=9. + * + * ________________________Start + * R0 = 0 + * R1 = r1 + * R2 = r2 + * ,------- JMP +4 * 3______________Preamble: 4 insns + * ,----------|-ind 0- if R0 != 7 JMP 8 * 3 + 1 <--------------------. + * | | R0 = 8 | + * | | JMP +7 * 3 ------------------------. + * | ,--------|-----1- if R0 != 5 JMP 7 * 3 + 1 <--------------. | | + * | | | R0 = 6 | | | + * | | | JMP +5 * 3 ------------------. | | + * | | ,------|-----2- if R0 != 3 JMP 6 * 3 + 1 <--------. | | | | + * | | | | R0 = 4 | | | | | + * | | | | JMP +3 * 3 ------------. | | | | + * | | | ,----|-----3- if R0 != 1 JMP 5 * 3 + 1 <--. | | | | | | + * | | | | | R0 = 2 | | | | | | | + * | | | | | JMP +1 * 3 ------. | | | | | | + * | | | | ,--t=====4> if R0 != 0 JMP 4 * 3 + 1 1 2 3 4 5 6 7 8 loc + * | | | | | R0 = 1 -1 +2 -3 +4 -5 +6 -7 +8 off + * | | | | | JMP -2 * 3 ---' | | | | | | | + * | | | | | ,------5- if R0 != 2 JMP 3 * 3 + 1 <-----' | | | | | | + * | | | | | | R0 = 3 | | | | | | + * | | | | | | JMP -4 * 3 ---------' | | | | | + * | | | | | | ,----6- if R0 != 4 JMP 2 * 3 + 1 <-----------' | | | | + * | | | | | | | R0 = 5 | | | | + * | | | | | | | JMP -6 * 3 ---------------' | | | + * | | | | | | | ,--7- if R0 != 6 JMP 1 * 3 + 1 <-----------------' | | + * | | | | | | | | R0 = 7 | | + * | | Error | | | JMP -8 * 3 ---------------------' | + * | | paths | | | ,8- if R0 != 8 JMP 0 * 3 + 1 <-----------------------' + * | | | | | | | | | R0 = 9__________________Sequence: 3 * size - 1 insns + * `-+-+-+-+-+-+-+-+-> EXIT____________________Return: 1 insn + * + */ + +/* The maximum size parameter */ +#define MAX_STAGGERED_JMP_SIZE ((0x7fff / 3) & ~1) + +/* We use a reduced number of iterations to get a reasonable execution time */ +#define NR_STAGGERED_JMP_RUNS 10 + +static int __bpf_fill_staggered_jumps(struct bpf_test *self, + const struct bpf_insn *jmp, + u64 r1, u64 r2) +{ + int size = self->test[0].result - 1; + int len = 4 + 3 * (size + 1); + struct bpf_insn *insns; + int off, ind; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Preamble */ + insns[0] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + insns[1] = BPF_ALU64_IMM(BPF_MOV, R1, r1); + insns[2] = BPF_ALU64_IMM(BPF_MOV, R2, r2); + insns[3] = BPF_JMP_IMM(BPF_JA, 0, 0, 3 * size / 2); + + /* Sequence */ + for (ind = 0, off = size; ind <= size; ind++, off -= 2) { + struct bpf_insn *ins = &insns[4 + 3 * ind]; + int loc; + + if (off == 0) + off--; + + loc = abs(off); + ins[0] = BPF_JMP_IMM(BPF_JNE, R0, loc - 1, + 3 * (size - ind) + 1); + ins[1] = BPF_ALU64_IMM(BPF_MOV, R0, loc); + ins[2] = *jmp; + ins[2].off = 3 * (off - 1); + } + + /* Return */ + insns[len - 1] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insns; + self->u.ptr.len = len; + + return 0; +} + +/* 64-bit unconditional jump */ +static int bpf_fill_staggered_ja(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JA, 0, 0, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0, 0); +} + +/* 64-bit immediate jumps */ +static int bpf_fill_staggered_jeq_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JEQ, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jne_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JNE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); +} + +static int bpf_fill_staggered_jset_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSET, R1, 0x82, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); +} + +static int bpf_fill_staggered_jgt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGT, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); +} + +static int bpf_fill_staggered_jge_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jlt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLT, R1, 0x80000000, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jle_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jsgt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGT, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +static int bpf_fill_staggered_jsge_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGE, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jslt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLT, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jsle_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLE, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +/* 64-bit register jumps */ +static int bpf_fill_staggered_jeq_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JEQ, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jne_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JNE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); +} + +static int bpf_fill_staggered_jset_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSET, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); +} + +static int bpf_fill_staggered_jgt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); +} + +static int bpf_fill_staggered_jge_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jlt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); +} + +static int bpf_fill_staggered_jle_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jsgt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); +} + +static int bpf_fill_staggered_jsge_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); +} + +static int bpf_fill_staggered_jslt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); +} + +static int bpf_fill_staggered_jsle_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); +} + +/* 32-bit immediate jumps */ +static int bpf_fill_staggered_jeq32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JEQ, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jne32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JNE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); +} + +static int bpf_fill_staggered_jset32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSET, R1, 0x82, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); +} + +static int bpf_fill_staggered_jgt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGT, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); +} + +static int bpf_fill_staggered_jge32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jlt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLT, R1, 0x80000000, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jle32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jsgt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGT, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +static int bpf_fill_staggered_jsge32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGE, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jslt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLT, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jsle32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLE, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +/* 32-bit register jumps */ +static int bpf_fill_staggered_jeq32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JEQ, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jne32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JNE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); +} + +static int bpf_fill_staggered_jset32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSET, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); +} + +static int bpf_fill_staggered_jgt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); +} + +static int bpf_fill_staggered_jge32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jlt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); +} + +static int bpf_fill_staggered_jle32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jsgt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); +} + +static int bpf_fill_staggered_jsge32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); +} + +static int bpf_fill_staggered_jslt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); +} + +static int bpf_fill_staggered_jsle32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); +} + + static struct bpf_test tests[] = { { "TAX", @@ -1095,7 +3693,7 @@ static struct bpf_test tests[] = { { "RET_A", .u.insns = { - /* check that unitialized X and A contain zeros */ + /* check that uninitialized X and A contain zeros */ BPF_STMT(BPF_MISC | BPF_TXA, 0), BPF_STMT(BPF_RET | BPF_A, 0) }, @@ -1916,6 +4514,22 @@ static struct bpf_test tests[] = { { }, { { 0, -1 } } }, +#ifdef CONFIG_32BIT + { + "INT: 32-bit context pointer word order and zero-extension", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_JMP32_IMM(BPF_JEQ, R1, 0, 3), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_JMP32_IMM(BPF_JNE, R1, 0, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, +#endif { "check: missing ret", .u.insns = { @@ -2361,6 +4975,48 @@ static struct bpf_test tests[] = { { { 0, 0x1 } }, }, { + "ALU_MOV_K: small negative", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "ALU_MOV_K: small negative zero extension", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU_MOV_K: large negative", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123456789), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123456789 } } + }, + { + "ALU_MOV_K: large negative zero extension", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123456789), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { "ALU64_MOV_K: dst = 2", .u.insns_int = { BPF_ALU64_IMM(BPF_MOV, R0, 2), @@ -2412,6 +5068,48 @@ static struct bpf_test tests[] = { { }, { { 0, 0x1 } }, }, + { + "ALU64_MOV_K: small negative", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, -123), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "ALU64_MOV_K: small negative sign extension", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, -123), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xffffffff } } + }, + { + "ALU64_MOV_K: large negative", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, -123456789), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123456789 } } + }, + { + "ALU64_MOV_K: large negative sign extension", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, -123456789), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xffffffff } } + }, /* BPF_ALU | BPF_ADD | BPF_X */ { "ALU_ADD_X: 1 + 2 = 3", @@ -2967,6 +5665,31 @@ static struct bpf_test tests[] = { { }, { { 0, 2147483647 } }, }, + { + "ALU64_MUL_X: 64x64 multiply, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0fedcba987654321LL), + BPF_LD_IMM64(R1, 0x123456789abcdef0LL), + BPF_ALU64_REG(BPF_MUL, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xe5618cf0 } } + }, + { + "ALU64_MUL_X: 64x64 multiply, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0fedcba987654321LL), + BPF_LD_IMM64(R1, 0x123456789abcdef0LL), + BPF_ALU64_REG(BPF_MUL, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x2236d88f } } + }, /* BPF_ALU | BPF_MUL | BPF_K */ { "ALU_MUL_K: 2 * 3 = 6", @@ -3077,6 +5800,29 @@ static struct bpf_test tests[] = { { }, { { 0, 0x1 } }, }, + { + "ALU64_MUL_K: 64x32 multiply, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_MUL, R0, 0x12345678), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xe242d208 } } + }, + { + "ALU64_MUL_K: 64x32 multiply, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_MUL, R0, 0x12345678), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xc28f5c28 } } + }, /* BPF_ALU | BPF_DIV | BPF_X */ { "ALU_DIV_X: 6 / 2 = 3", @@ -3431,6 +6177,44 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, }, { + "ALU_AND_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), + BPF_ALU32_IMM(BPF_AND, R0, 15), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 4 } } + }, + { + "ALU_AND_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xf1f2f3f4), + BPF_ALU32_IMM(BPF_AND, R0, 0xafbfcfdf), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xa1b2c3d4 } } + }, + { + "ALU_AND_K: Zero extension", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x0000000080a0c0e0LL), + BPF_ALU32_IMM(BPF_AND, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { "ALU64_AND_K: 3 & 2 = 2", .u.insns_int = { BPF_LD_IMM64(R0, 3), @@ -3453,7 +6237,7 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, }, { - "ALU64_AND_K: 0x0000ffffffff0000 & 0x0 = 0x0000ffff00000000", + "ALU64_AND_K: 0x0000ffffffff0000 & 0x0 = 0x0000000000000000", .u.insns_int = { BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), BPF_LD_IMM64(R3, 0x0000000000000000LL), @@ -3469,7 +6253,7 @@ static struct bpf_test tests[] = { { { 0, 0x1 } }, }, { - "ALU64_AND_K: 0x0000ffffffff0000 & -1 = 0x0000ffffffffffff", + "ALU64_AND_K: 0x0000ffffffff0000 & -1 = 0x0000ffffffff0000", .u.insns_int = { BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), @@ -3500,6 +6284,38 @@ static struct bpf_test tests[] = { { }, { { 0, 0x1 } }, }, + { + "ALU64_AND_K: Sign extension 1", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x00000000090b0d0fLL), + BPF_ALU64_IMM(BPF_AND, R0, 0x0f0f0f0f), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { + "ALU64_AND_K: Sign extension 2", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x0123456780a0c0e0LL), + BPF_ALU64_IMM(BPF_AND, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, /* BPF_ALU | BPF_OR | BPF_X */ { "ALU_OR_X: 1 | 2 = 3", @@ -3573,6 +6389,44 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, }, { + "ALU_OR_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), + BPF_ALU32_IMM(BPF_OR, R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x01020305 } } + }, + { + "ALU_OR_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), + BPF_ALU32_IMM(BPF_OR, R0, 0xa0b0c0d0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xa1b2c3d4 } } + }, + { + "ALU_OR_K: Zero extension", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x00000000f9fbfdffLL), + BPF_ALU32_IMM(BPF_OR, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { "ALU64_OR_K: 1 | 2 = 3", .u.insns_int = { BPF_LD_IMM64(R0, 1), @@ -3595,7 +6449,7 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, }, { - "ALU64_OR_K: 0x0000ffffffff0000 | 0x0 = 0x0000ffff00000000", + "ALU64_OR_K: 0x0000ffffffff0000 | 0x0 = 0x0000ffffffff0000", .u.insns_int = { BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), @@ -3642,6 +6496,38 @@ static struct bpf_test tests[] = { { }, { { 0, 0x1 } }, }, + { + "ALU64_OR_K: Sign extension 1", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x012345678fafcfefLL), + BPF_ALU64_IMM(BPF_OR, R0, 0x0f0f0f0f), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { + "ALU64_OR_K: Sign extension 2", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0xfffffffff9fbfdffLL), + BPF_ALU64_IMM(BPF_OR, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, /* BPF_ALU | BPF_XOR | BPF_X */ { "ALU_XOR_X: 5 ^ 6 = 3", @@ -3715,6 +6601,44 @@ static struct bpf_test tests[] = { { { 0, 0xfffffffe } }, }, { + "ALU_XOR_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), + BPF_ALU32_IMM(BPF_XOR, R0, 15), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x0102030b } } + }, + { + "ALU_XOR_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xf1f2f3f4), + BPF_ALU32_IMM(BPF_XOR, R0, 0xafbfcfdf), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x5e4d3c2b } } + }, + { + "ALU_XOR_K: Zero extension", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x00000000795b3d1fLL), + BPF_ALU32_IMM(BPF_XOR, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { "ALU64_XOR_K: 5 ^ 6 = 3", .u.insns_int = { BPF_LD_IMM64(R0, 5), @@ -3726,7 +6650,7 @@ static struct bpf_test tests[] = { { { 0, 3 } }, }, { - "ALU64_XOR_K: 1 & 0xffffffff = 0xfffffffe", + "ALU64_XOR_K: 1 ^ 0xffffffff = 0xfffffffe", .u.insns_int = { BPF_LD_IMM64(R0, 1), BPF_ALU64_IMM(BPF_XOR, R0, 0xffffffff), @@ -3784,6 +6708,38 @@ static struct bpf_test tests[] = { { }, { { 0, 0x1 } }, }, + { + "ALU64_XOR_K: Sign extension 1", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x0123456786a4c2e0LL), + BPF_ALU64_IMM(BPF_XOR, R0, 0x0f0f0f0f), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { + "ALU64_XOR_K: Sign extension 2", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0xfedcba98795b3d1fLL), + BPF_ALU64_IMM(BPF_XOR, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, /* BPF_ALU | BPF_LSH | BPF_X */ { "ALU_LSH_X: 1 << 1 = 2", @@ -3810,6 +6766,18 @@ static struct bpf_test tests[] = { { { 0, 0x80000000 } }, }, { + "ALU_LSH_X: 0x12345678 << 12 = 0x45678000", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU32_REG(BPF_LSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x45678000 } } + }, + { "ALU64_LSH_X: 1 << 1 = 2", .u.insns_int = { BPF_LD_IMM64(R0, 1), @@ -3833,6 +6801,106 @@ static struct bpf_test tests[] = { { }, { { 0, 0x80000000 } }, }, + { + "ALU64_LSH_X: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xbcdef000 } } + }, + { + "ALU64_LSH_X: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x3456789a } } + }, + { + "ALU64_LSH_X: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_LSH_X: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x9abcdef0 } } + }, + { + "ALU64_LSH_X: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_LSH_X: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, + { + "ALU64_LSH_X: Zero shift, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, + { + "ALU64_LSH_X: Zero shift, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x01234567 } } + }, /* BPF_ALU | BPF_LSH | BPF_K */ { "ALU_LSH_K: 1 << 1 = 2", @@ -3857,6 +6925,28 @@ static struct bpf_test tests[] = { { { 0, 0x80000000 } }, }, { + "ALU_LSH_K: 0x12345678 << 12 = 0x45678000", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_LSH, R0, 12), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x45678000 } } + }, + { + "ALU_LSH_K: 0x12345678 << 0 = 0x12345678", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_LSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x12345678 } } + }, + { "ALU64_LSH_K: 1 << 1 = 2", .u.insns_int = { BPF_LD_IMM64(R0, 1), @@ -3878,6 +6968,86 @@ static struct bpf_test tests[] = { { }, { { 0, 0x80000000 } }, }, + { + "ALU64_LSH_K: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 12), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xbcdef000 } } + }, + { + "ALU64_LSH_K: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 12), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x3456789a } } + }, + { + "ALU64_LSH_K: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 36), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_LSH_K: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 36), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x9abcdef0 } } + }, + { + "ALU64_LSH_K: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_LSH_K: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 32), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, + { + "ALU64_LSH_K: Zero shift", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, /* BPF_ALU | BPF_RSH | BPF_X */ { "ALU_RSH_X: 2 >> 1 = 1", @@ -3904,6 +7074,18 @@ static struct bpf_test tests[] = { { { 0, 1 } }, }, { + "ALU_RSH_X: 0x12345678 >> 20 = 0x123", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_MOV, R1, 20), + BPF_ALU32_REG(BPF_RSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x123 } } + }, + { "ALU64_RSH_X: 2 >> 1 = 1", .u.insns_int = { BPF_LD_IMM64(R0, 2), @@ -3927,6 +7109,106 @@ static struct bpf_test tests[] = { { }, { { 0, 1 } }, }, + { + "ALU64_RSH_X: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x56789abc } } + }, + { + "ALU64_RSH_X: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x00081234 } } + }, + { + "ALU64_RSH_X: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x08123456 } } + }, + { + "ALU64_RSH_X: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_RSH_X: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, + { + "ALU64_RSH_X: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_RSH_X: Zero shift, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, + { + "ALU64_RSH_X: Zero shift, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, /* BPF_ALU | BPF_RSH | BPF_K */ { "ALU_RSH_K: 2 >> 1 = 1", @@ -3951,6 +7233,28 @@ static struct bpf_test tests[] = { { { 0, 1 } }, }, { + "ALU_RSH_K: 0x12345678 >> 20 = 0x123", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_RSH, R0, 20), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x123 } } + }, + { + "ALU_RSH_K: 0x12345678 >> 0 = 0x12345678", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_RSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x12345678 } } + }, + { "ALU64_RSH_K: 2 >> 1 = 1", .u.insns_int = { BPF_LD_IMM64(R0, 2), @@ -3972,9 +7276,101 @@ static struct bpf_test tests[] = { { }, { { 0, 1 } }, }, + { + "ALU64_RSH_K: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 12), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x56789abc } } + }, + { + "ALU64_RSH_K: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 12), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x00081234 } } + }, + { + "ALU64_RSH_K: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 36), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x08123456 } } + }, + { + "ALU64_RSH_K: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 36), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_RSH_K: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, + { + "ALU64_RSH_K: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_RSH_K: Zero shift", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, /* BPF_ALU | BPF_ARSH | BPF_X */ { - "ALU_ARSH_X: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", + "ALU32_ARSH_X: -1234 >> 7 = -10", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -1234), + BPF_ALU32_IMM(BPF_MOV, R1, 7), + BPF_ALU32_REG(BPF_ARSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -10 } } + }, + { + "ALU64_ARSH_X: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", .u.insns_int = { BPF_LD_IMM64(R0, 0xff00ff0000000000LL), BPF_ALU32_IMM(BPF_MOV, R1, 40), @@ -3985,9 +7381,131 @@ static struct bpf_test tests[] = { { }, { { 0, 0xffff00ff } }, }, + { + "ALU64_ARSH_X: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x56789abc } } + }, + { + "ALU64_ARSH_X: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfff81234 } } + }, + { + "ALU64_ARSH_X: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xf8123456 } } + }, + { + "ALU64_ARSH_X: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } } + }, + { + "ALU64_ARSH_X: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, + { + "ALU64_ARSH_X: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } } + }, + { + "ALU64_ARSH_X: Zero shift, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, + { + "ALU64_ARSH_X: Zero shift, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, /* BPF_ALU | BPF_ARSH | BPF_K */ { - "ALU_ARSH_K: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", + "ALU32_ARSH_K: -1234 >> 7 = -10", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -1234), + BPF_ALU32_IMM(BPF_ARSH, R0, 7), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -10 } } + }, + { + "ALU32_ARSH_K: -1234 >> 0 = -1234", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -1234), + BPF_ALU32_IMM(BPF_ARSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1234 } } + }, + { + "ALU64_ARSH_K: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", .u.insns_int = { BPF_LD_IMM64(R0, 0xff00ff0000000000LL), BPF_ALU64_IMM(BPF_ARSH, R0, 40), @@ -3997,6 +7515,86 @@ static struct bpf_test tests[] = { { }, { { 0, 0xffff00ff } }, }, + { + "ALU64_ARSH_K: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 12), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x56789abc } } + }, + { + "ALU64_ARSH_K: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 12), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfff81234 } } + }, + { + "ALU64_ARSH_K: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 36), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xf8123456 } } + }, + { + "ALU64_ARSH_K: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xf123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 36), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } } + }, + { + "ALU64_ARSH_K: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, + { + "ALU64_ARSH_K: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 32), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } } + }, + { + "ALU64_ARSH_K: Zero shift", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, /* BPF_ALU | BPF_NEG */ { "ALU_NEG: -(3) = -3", @@ -4079,6 +7677,67 @@ static struct bpf_test tests[] = { { }, { { 0, (u32) cpu_to_be64(0x0123456789abcdefLL) } }, }, + { + "ALU_END_FROM_BE 64: 0x0123456789abcdef >> 32 -> 0x01234567", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ENDIAN(BPF_FROM_BE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_be64(0x0123456789abcdefLL) >> 32) } }, + }, + /* BPF_ALU | BPF_END | BPF_FROM_BE, reversed */ + { + "ALU_END_FROM_BE 16: 0xfedcba9876543210 -> 0x3210", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 16), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_be16(0x3210) } }, + }, + { + "ALU_END_FROM_BE 32: 0xfedcba9876543210 -> 0x76543210", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 32), + BPF_ALU64_REG(BPF_MOV, R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_be32(0x76543210) } }, + }, + { + "ALU_END_FROM_BE 64: 0xfedcba9876543210 -> 0x76543210", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 64), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) cpu_to_be64(0xfedcba9876543210ULL) } }, + }, + { + "ALU_END_FROM_BE 64: 0xfedcba9876543210 >> 32 -> 0xfedcba98", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_be64(0xfedcba9876543210ULL) >> 32) } }, + }, /* BPF_ALU | BPF_END | BPF_FROM_LE */ { "ALU_END_FROM_LE 16: 0x0123456789abcdef -> 0xefcd", @@ -4116,6 +7775,321 @@ static struct bpf_test tests[] = { { }, { { 0, (u32) cpu_to_le64(0x0123456789abcdefLL) } }, }, + { + "ALU_END_FROM_LE 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ENDIAN(BPF_FROM_LE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_le64(0x0123456789abcdefLL) >> 32) } }, + }, + /* BPF_ALU | BPF_END | BPF_FROM_LE, reversed */ + { + "ALU_END_FROM_LE 16: 0xfedcba9876543210 -> 0x1032", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 16), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_le16(0x3210) } }, + }, + { + "ALU_END_FROM_LE 32: 0xfedcba9876543210 -> 0x10325476", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 32), + BPF_ALU64_REG(BPF_MOV, R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_le32(0x76543210) } }, + }, + { + "ALU_END_FROM_LE 64: 0xfedcba9876543210 -> 0x10325476", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 64), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) cpu_to_le64(0xfedcba9876543210ULL) } }, + }, + { + "ALU_END_FROM_LE 64: 0xfedcba9876543210 >> 32 -> 0x98badcfe", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_le64(0xfedcba9876543210ULL) >> 32) } }, + }, + /* BPF_LDX_MEM B/H/W/DW */ + { + "BPF_LDX_MEM | BPF_B", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0102030405060708ULL), + BPF_LD_IMM64(R2, 0x0000000000000008ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_B, R0, R10, -1), +#else + BPF_LDX_MEM(BPF_B, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_B, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8182838485868788ULL), + BPF_LD_IMM64(R2, 0x0000000000000088ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_B, R0, R10, -1), +#else + BPF_LDX_MEM(BPF_B, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_H", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0102030405060708ULL), + BPF_LD_IMM64(R2, 0x0000000000000708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_H, R0, R10, -2), +#else + BPF_LDX_MEM(BPF_H, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_H, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8182838485868788ULL), + BPF_LD_IMM64(R2, 0x0000000000008788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_H, R0, R10, -2), +#else + BPF_LDX_MEM(BPF_H, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_W", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0102030405060708ULL), + BPF_LD_IMM64(R2, 0x0000000005060708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_W, R0, R10, -4), +#else + BPF_LDX_MEM(BPF_W, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_W, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8182838485868788ULL), + BPF_LD_IMM64(R2, 0x0000000085868788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_W, R0, R10, -4), +#else + BPF_LDX_MEM(BPF_W, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + /* BPF_STX_MEM B/H/W/DW */ + { + "BPF_STX_MEM | BPF_B", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x0102030405060708ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d0e008ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_B, R10, R2, -1), +#else + BPF_STX_MEM(BPF_B, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_B, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d0e088ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_B, R10, R2, -1), +#else + BPF_STX_MEM(BPF_B, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_H", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x0102030405060708ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d00708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_H, R10, R2, -2), +#else + BPF_STX_MEM(BPF_H, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_H, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d08788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_H, R10, R2, -2), +#else + BPF_STX_MEM(BPF_H, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_W", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x0102030405060708ULL), + BPF_LD_IMM64(R3, 0x8090a0b005060708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_W, R10, R2, -4), +#else + BPF_STX_MEM(BPF_W, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_W, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x8090a0b085868788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_W, R10, R2, -4), +#else + BPF_STX_MEM(BPF_W, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, /* BPF_ST(X) | BPF_MEM | BPF_B/H/W/DW */ { "ST_MEM_B: Store/Load byte: max negative", @@ -4286,8 +8260,8 @@ static struct bpf_test tests[] = { .u.insns_int = { BPF_LD_IMM64(R0, 0), BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_STX_MEM(BPF_W, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_LDX_MEM(BPF_DW, R0, R10, -40), BPF_EXIT_INSN(), }, INTERNAL, @@ -4295,80 +8269,358 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, .stack_depth = 40, }, - /* BPF_STX | BPF_XADD | BPF_W/DW */ { - "STX_XADD_W: Test: 0x12 + 0x10 = 0x22", + "STX_MEM_DW: Store double word: first word in memory", .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_W, R10, -40, 0x10), - BPF_STX_XADD(BPF_W, R10, R0, -40), + BPF_LD_IMM64(R0, 0), + BPF_LD_IMM64(R1, 0x0123456789abcdefLL), + BPF_STX_MEM(BPF_DW, R10, R1, -40), BPF_LDX_MEM(BPF_W, R0, R10, -40), BPF_EXIT_INSN(), }, INTERNAL, { }, - { { 0, 0x22 } }, +#ifdef __BIG_ENDIAN + { { 0, 0x01234567 } }, +#else + { { 0, 0x89abcdef } }, +#endif .stack_depth = 40, }, { - "STX_XADD_W: Test side-effects, r10: 0x12 + 0x10 = 0x22", + "STX_MEM_DW: Store double word: second word in memory", .u.insns_int = { - BPF_ALU64_REG(BPF_MOV, R1, R10), - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_W, R10, -40, 0x10), - BPF_STX_XADD(BPF_W, R10, R0, -40), - BPF_ALU64_REG(BPF_MOV, R0, R10), - BPF_ALU64_REG(BPF_SUB, R0, R1), + BPF_LD_IMM64(R0, 0), + BPF_LD_IMM64(R1, 0x0123456789abcdefLL), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_LDX_MEM(BPF_W, R0, R10, -36), BPF_EXIT_INSN(), }, INTERNAL, { }, - { { 0, 0 } }, +#ifdef __BIG_ENDIAN + { { 0, 0x89abcdef } }, +#else + { { 0, 0x01234567 } }, +#endif .stack_depth = 40, }, + /* BPF_STX | BPF_ATOMIC | BPF_W/DW */ + { + "STX_XADD_W: X + 1 + 1 + 1 + ...", + { }, + INTERNAL, + { }, + { { 0, 4134 } }, + .fill_helper = bpf_fill_stxw, + }, { - "STX_XADD_W: Test side-effects, r0: 0x12 + 0x10 = 0x22", + "STX_XADD_DW: X + 1 + 1 + 1 + ...", + { }, + INTERNAL, + { }, + { { 0, 4134 } }, + .fill_helper = bpf_fill_stxdw, + }, + /* + * Exhaustive tests of atomic operation variants. + * Individual tests are expanded from template macros for all + * combinations of ALU operation, word size and fetching. + */ +#define BPF_ATOMIC_POISON(width) ((width) == BPF_W ? (0xbaadf00dULL << 32) : 0) + +#define BPF_ATOMIC_OP_TEST1(width, op, logic, old, update, result) \ +{ \ + "BPF_ATOMIC | " #width ", " #op ": Test: " \ + #old " " #logic " " #update " = " #result, \ + .u.insns_int = { \ + BPF_LD_IMM64(R5, (update) | BPF_ATOMIC_POISON(width)), \ + BPF_ST_MEM(width, R10, -40, old), \ + BPF_ATOMIC_OP(width, op, R10, R5, -40), \ + BPF_LDX_MEM(width, R0, R10, -40), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, result } }, \ + .stack_depth = 40, \ +} +#define BPF_ATOMIC_OP_TEST2(width, op, logic, old, update, result) \ +{ \ + "BPF_ATOMIC | " #width ", " #op ": Test side effects, r10: " \ + #old " " #logic " " #update " = " #result, \ + .u.insns_int = { \ + BPF_ALU64_REG(BPF_MOV, R1, R10), \ + BPF_LD_IMM64(R0, (update) | BPF_ATOMIC_POISON(width)), \ + BPF_ST_MEM(BPF_W, R10, -40, old), \ + BPF_ATOMIC_OP(width, op, R10, R0, -40), \ + BPF_ALU64_REG(BPF_MOV, R0, R10), \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + .stack_depth = 40, \ +} +#define BPF_ATOMIC_OP_TEST3(width, op, logic, old, update, result) \ +{ \ + "BPF_ATOMIC | " #width ", " #op ": Test side effects, r0: " \ + #old " " #logic " " #update " = " #result, \ + .u.insns_int = { \ + BPF_ALU64_REG(BPF_MOV, R0, R10), \ + BPF_LD_IMM64(R1, (update) | BPF_ATOMIC_POISON(width)), \ + BPF_ST_MEM(width, R10, -40, old), \ + BPF_ATOMIC_OP(width, op, R10, R1, -40), \ + BPF_ALU64_REG(BPF_SUB, R0, R10), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + .stack_depth = 40, \ +} +#define BPF_ATOMIC_OP_TEST4(width, op, logic, old, update, result) \ +{ \ + "BPF_ATOMIC | " #width ", " #op ": Test fetch: " \ + #old " " #logic " " #update " = " #result, \ + .u.insns_int = { \ + BPF_LD_IMM64(R3, (update) | BPF_ATOMIC_POISON(width)), \ + BPF_ST_MEM(width, R10, -40, old), \ + BPF_ATOMIC_OP(width, op, R10, R3, -40), \ + BPF_ALU32_REG(BPF_MOV, R0, R3), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, (op) & BPF_FETCH ? old : update } }, \ + .stack_depth = 40, \ +} + /* BPF_ATOMIC | BPF_W: BPF_ADD */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), + /* BPF_ATOMIC | BPF_W: BPF_ADD | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + /* BPF_ATOMIC | BPF_DW: BPF_ADD */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), + /* BPF_ATOMIC | BPF_DW: BPF_ADD | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + /* BPF_ATOMIC | BPF_W: BPF_AND */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), + /* BPF_ATOMIC | BPF_W: BPF_AND | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + /* BPF_ATOMIC | BPF_DW: BPF_AND */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), + /* BPF_ATOMIC | BPF_DW: BPF_AND | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + /* BPF_ATOMIC | BPF_W: BPF_OR */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), + /* BPF_ATOMIC | BPF_W: BPF_OR | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + /* BPF_ATOMIC | BPF_DW: BPF_OR */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), + /* BPF_ATOMIC | BPF_DW: BPF_OR | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + /* BPF_ATOMIC | BPF_W: BPF_XOR */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), + /* BPF_ATOMIC | BPF_W: BPF_XOR | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + /* BPF_ATOMIC | BPF_DW: BPF_XOR */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), + /* BPF_ATOMIC | BPF_DW: BPF_XOR | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + /* BPF_ATOMIC | BPF_W: BPF_XCHG */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + /* BPF_ATOMIC | BPF_DW: BPF_XCHG */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), +#undef BPF_ATOMIC_POISON +#undef BPF_ATOMIC_OP_TEST1 +#undef BPF_ATOMIC_OP_TEST2 +#undef BPF_ATOMIC_OP_TEST3 +#undef BPF_ATOMIC_OP_TEST4 + /* BPF_ATOMIC | BPF_W, BPF_CMPXCHG */ + { + "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test successful return", + .u.insns_int = { + BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x01234567 } }, + .stack_depth = 40, + }, + { + "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test successful store", .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_W, R10, -40, 0x10), - BPF_STX_XADD(BPF_W, R10, R0, -40), + BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), + BPF_LDX_MEM(BPF_W, R0, R10, -40), BPF_EXIT_INSN(), }, INTERNAL, { }, - { { 0, 0x12 } }, + { { 0, 0x89abcdef } }, .stack_depth = 40, }, { - "STX_XADD_W: X + 1 + 1 + 1 + ...", + "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test failure return", + .u.insns_int = { + BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R0, 0x76543210), + BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), + BPF_EXIT_INSN(), + }, + INTERNAL, { }, + { { 0, 0x01234567 } }, + .stack_depth = 40, + }, + { + "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test failure store", + .u.insns_int = { + BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R0, 0x76543210), + BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), + BPF_LDX_MEM(BPF_W, R0, R10, -40), + BPF_EXIT_INSN(), + }, INTERNAL, { }, - { { 0, 4134 } }, - .fill_helper = bpf_fill_stxw, + { { 0, 0x01234567 } }, + .stack_depth = 40, }, { - "STX_XADD_DW: Test: 0x12 + 0x10 = 0x22", + "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test side effects", .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_DW, R10, -40, 0x10), - BPF_STX_XADD(BPF_DW, R10, R0, -40), + BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), + BPF_ALU32_REG(BPF_MOV, R0, R3), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } }, + .stack_depth = 40, + }, + /* BPF_ATOMIC | BPF_DW, BPF_CMPXCHG */ + { + "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful return", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0123456789abcdefULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), + BPF_JMP_REG(BPF_JNE, R0, R1, 1), + BPF_ALU64_REG(BPF_SUB, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 40, + }, + { + "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful store", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0123456789abcdefULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_STX_MEM(BPF_DW, R10, R0, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), BPF_LDX_MEM(BPF_DW, R0, R10, -40), + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_REG(BPF_SUB, R0, R2), BPF_EXIT_INSN(), }, INTERNAL, { }, - { { 0, 0x22 } }, + { { 0, 0 } }, .stack_depth = 40, }, { - "STX_XADD_DW: Test side-effects, r10: 0x12 + 0x10 = 0x22", + "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure return", .u.insns_int = { - BPF_ALU64_REG(BPF_MOV, R1, R10), - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_DW, R10, -40, 0x10), - BPF_STX_XADD(BPF_DW, R10, R0, -40), - BPF_ALU64_REG(BPF_MOV, R0, R10), + BPF_LD_IMM64(R1, 0x0123456789abcdefULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_ALU64_IMM(BPF_ADD, R0, 1), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), + BPF_JMP_REG(BPF_JNE, R0, R1, 1), BPF_ALU64_REG(BPF_SUB, R0, R1), BPF_EXIT_INSN(), }, @@ -4378,25 +8630,552 @@ static struct bpf_test tests[] = { .stack_depth = 40, }, { - "STX_XADD_DW: Test side-effects, r0: 0x12 + 0x10 = 0x22", + "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure store", .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_DW, R10, -40, 0x10), - BPF_STX_XADD(BPF_DW, R10, R0, -40), + BPF_LD_IMM64(R1, 0x0123456789abcdefULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_ALU64_IMM(BPF_ADD, R0, 1), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), + BPF_LDX_MEM(BPF_DW, R0, R10, -40), + BPF_JMP_REG(BPF_JNE, R0, R1, 1), + BPF_ALU64_REG(BPF_SUB, R0, R1), BPF_EXIT_INSN(), }, INTERNAL, { }, - { { 0, 0x12 } }, + { { 0, 0 } }, .stack_depth = 40, }, { - "STX_XADD_DW: X + 1 + 1 + 1 + ...", + "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test side effects", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0123456789abcdefULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_REG(BPF_SUB, R0, R2), + BPF_EXIT_INSN(), + }, + INTERNAL, { }, + { { 0, 0 } }, + .stack_depth = 40, + }, + /* BPF_JMP32 | BPF_JEQ | BPF_K */ + { + "JMP32_JEQ_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JEQ, R0, 321, 1), + BPF_JMP32_IMM(BPF_JEQ, R0, 123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, INTERNAL, { }, - { { 0, 4134 } }, - .fill_helper = bpf_fill_stxdw, + { { 0, 123 } } + }, + { + "JMP32_JEQ_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 12345678), + BPF_JMP32_IMM(BPF_JEQ, R0, 12345678 & 0xffff, 1), + BPF_JMP32_IMM(BPF_JEQ, R0, 12345678, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 12345678 } } + }, + { + "JMP32_JEQ_K: negative immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JEQ, R0, 123, 1), + BPF_JMP32_IMM(BPF_JEQ, R0, -123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + /* BPF_JMP32 | BPF_JEQ | BPF_X */ + { + "JMP32_JEQ_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 1234), + BPF_ALU32_IMM(BPF_MOV, R1, 4321), + BPF_JMP32_REG(BPF_JEQ, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 1234), + BPF_JMP32_REG(BPF_JEQ, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1234 } } + }, + /* BPF_JMP32 | BPF_JNE | BPF_K */ + { + "JMP32_JNE_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JNE, R0, 123, 1), + BPF_JMP32_IMM(BPF_JNE, R0, 321, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } } + }, + { + "JMP32_JNE_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 12345678), + BPF_JMP32_IMM(BPF_JNE, R0, 12345678, 1), + BPF_JMP32_IMM(BPF_JNE, R0, 12345678 & 0xffff, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 12345678 } } + }, + { + "JMP32_JNE_K: negative immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JNE, R0, -123, 1), + BPF_JMP32_IMM(BPF_JNE, R0, 123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + /* BPF_JMP32 | BPF_JNE | BPF_X */ + { + "JMP32_JNE_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 1234), + BPF_ALU32_IMM(BPF_MOV, R1, 1234), + BPF_JMP32_REG(BPF_JNE, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 4321), + BPF_JMP32_REG(BPF_JNE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1234 } } + }, + /* BPF_JMP32 | BPF_JSET | BPF_K */ + { + "JMP32_JSET_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSET, R0, 2, 1), + BPF_JMP32_IMM(BPF_JSET, R0, 3, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { + "JMP32_JSET_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x40000000), + BPF_JMP32_IMM(BPF_JSET, R0, 0x3fffffff, 1), + BPF_JMP32_IMM(BPF_JSET, R0, 0x60000000, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x40000000 } } + }, + { + "JMP32_JSET_K: negative immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JSET, R0, -1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + /* BPF_JMP32 | BPF_JSET | BPF_X */ + { + "JMP32_JSET_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 8), + BPF_ALU32_IMM(BPF_MOV, R1, 7), + BPF_JMP32_REG(BPF_JSET, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 8 | 2), + BPF_JMP32_REG(BPF_JNE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 8 } } + }, + /* BPF_JMP32 | BPF_JGT | BPF_K */ + { + "JMP32_JGT_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JGT, R0, 123, 1), + BPF_JMP32_IMM(BPF_JGT, R0, 122, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } } + }, + { + "JMP32_JGT_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_JMP32_IMM(BPF_JGT, R0, 0xffffffff, 1), + BPF_JMP32_IMM(BPF_JGT, R0, 0xfffffffd, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JGT | BPF_X */ + { + "JMP32_JGT_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), + BPF_JMP32_REG(BPF_JGT, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), + BPF_JMP32_REG(BPF_JGT, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JGE | BPF_K */ + { + "JMP32_JGE_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JGE, R0, 124, 1), + BPF_JMP32_IMM(BPF_JGE, R0, 123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } } + }, + { + "JMP32_JGE_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_JMP32_IMM(BPF_JGE, R0, 0xffffffff, 1), + BPF_JMP32_IMM(BPF_JGE, R0, 0xfffffffe, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JGE | BPF_X */ + { + "JMP32_JGE_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), + BPF_JMP32_REG(BPF_JGE, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffe), + BPF_JMP32_REG(BPF_JGE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JLT | BPF_K */ + { + "JMP32_JLT_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JLT, R0, 123, 1), + BPF_JMP32_IMM(BPF_JLT, R0, 124, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } } + }, + { + "JMP32_JLT_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_JMP32_IMM(BPF_JLT, R0, 0xfffffffd, 1), + BPF_JMP32_IMM(BPF_JLT, R0, 0xffffffff, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JLT | BPF_X */ + { + "JMP32_JLT_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), + BPF_JMP32_REG(BPF_JLT, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), + BPF_JMP32_REG(BPF_JLT, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JLE | BPF_K */ + { + "JMP32_JLE_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JLE, R0, 122, 1), + BPF_JMP32_IMM(BPF_JLE, R0, 123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } } + }, + { + "JMP32_JLE_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_JMP32_IMM(BPF_JLE, R0, 0xfffffffd, 1), + BPF_JMP32_IMM(BPF_JLE, R0, 0xfffffffe, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JLE | BPF_X */ + { + "JMP32_JLE_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), + BPF_JMP32_REG(BPF_JLE, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffe), + BPF_JMP32_REG(BPF_JLE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JSGT | BPF_K */ + { + "JMP32_JSGT_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JSGT, R0, -123, 1), + BPF_JMP32_IMM(BPF_JSGT, R0, -124, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "JMP32_JSGT_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_JMP32_IMM(BPF_JSGT, R0, -12345678, 1), + BPF_JMP32_IMM(BPF_JSGT, R0, -12345679, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSGT | BPF_X */ + { + "JMP32_JSGT_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_ALU32_IMM(BPF_MOV, R1, -12345678), + BPF_JMP32_REG(BPF_JSGT, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, -12345679), + BPF_JMP32_REG(BPF_JSGT, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSGE | BPF_K */ + { + "JMP32_JSGE_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JSGE, R0, -122, 1), + BPF_JMP32_IMM(BPF_JSGE, R0, -123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "JMP32_JSGE_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_JMP32_IMM(BPF_JSGE, R0, -12345677, 1), + BPF_JMP32_IMM(BPF_JSGE, R0, -12345678, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSGE | BPF_X */ + { + "JMP32_JSGE_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_ALU32_IMM(BPF_MOV, R1, -12345677), + BPF_JMP32_REG(BPF_JSGE, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, -12345678), + BPF_JMP32_REG(BPF_JSGE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSLT | BPF_K */ + { + "JMP32_JSLT_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JSLT, R0, -123, 1), + BPF_JMP32_IMM(BPF_JSLT, R0, -122, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "JMP32_JSLT_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_JMP32_IMM(BPF_JSLT, R0, -12345678, 1), + BPF_JMP32_IMM(BPF_JSLT, R0, -12345677, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSLT | BPF_X */ + { + "JMP32_JSLT_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_ALU32_IMM(BPF_MOV, R1, -12345678), + BPF_JMP32_REG(BPF_JSLT, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, -12345677), + BPF_JMP32_REG(BPF_JSLT, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSLE | BPF_K */ + { + "JMP32_JSLE_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JSLE, R0, -124, 1), + BPF_JMP32_IMM(BPF_JSLE, R0, -123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "JMP32_JSLE_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_JMP32_IMM(BPF_JSLE, R0, -12345679, 1), + BPF_JMP32_IMM(BPF_JSLE, R0, -12345678, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSLE | BPF_K */ + { + "JMP32_JSLE_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_ALU32_IMM(BPF_MOV, R1, -12345679), + BPF_JMP32_REG(BPF_JSLE, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, -12345678), + BPF_JMP32_REG(BPF_JSLE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } }, /* BPF_JMP | BPF_EXIT */ { @@ -5275,31 +10054,21 @@ static struct bpf_test tests[] = { { /* Mainly checking JIT here. */ "BPF_MAXINSNS: Ctx heavy transformations", { }, -#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_S390) - CLASSIC | FLAG_EXPECTED_FAIL, -#else CLASSIC, -#endif { }, { { 1, SKB_VLAN_PRESENT }, { 10, SKB_VLAN_PRESENT } }, .fill_helper = bpf_fill_maxinsns6, - .expected_errcode = -ENOTSUPP, }, { /* Mainly checking JIT here. */ "BPF_MAXINSNS: Call heavy transformations", { }, -#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_S390) - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, -#else CLASSIC | FLAG_NO_DATA, -#endif { }, { { 1, 0 }, { 10, 0 } }, .fill_helper = bpf_fill_maxinsns7, - .expected_errcode = -ENOTSUPP, }, { /* Mainly checking JIT here. */ "BPF_MAXINSNS: Jump heavy test", @@ -5328,15 +10097,10 @@ static struct bpf_test tests[] = { { "BPF_MAXINSNS: Jump, gap, jump, ...", { }, -#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_X86) - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, -#else CLASSIC | FLAG_NO_DATA, -#endif { }, { { 0, 0xababcbac } }, .fill_helper = bpf_fill_maxinsns11, - .expected_errcode = -ENOTSUPP, }, { "BPF_MAXINSNS: jump over MSH", @@ -5350,28 +10114,18 @@ static struct bpf_test tests[] = { { "BPF_MAXINSNS: exec all MSH", { }, -#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_S390) - CLASSIC | FLAG_EXPECTED_FAIL, -#else CLASSIC, -#endif { 0xfa, 0xfb, 0xfc, 0xfd, }, { { 4, 0xababab83 } }, .fill_helper = bpf_fill_maxinsns13, - .expected_errcode = -ENOTSUPP, }, { "BPF_MAXINSNS: ld_abs+get_processor_id", { }, -#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_S390) - CLASSIC | FLAG_EXPECTED_FAIL, -#else CLASSIC, -#endif { }, { { 1, 0xbee } }, .fill_helper = bpf_fill_ld_abs_get_processor_id, - .expected_errcode = -ENOTSUPP, }, /* * LD_IND / LD_ABS on fragmented SKBs @@ -6461,6 +11215,2841 @@ static struct bpf_test tests[] = { {}, { { 0, 2 } }, }, + /* BPF_LDX_MEM with operand aliasing */ + { + "LDX_MEM_B: operand register aliasing", + .u.insns_int = { + BPF_ST_MEM(BPF_B, R10, -8, 123), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_B, R0, R0, -8), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } }, + .stack_depth = 8, + }, + { + "LDX_MEM_H: operand register aliasing", + .u.insns_int = { + BPF_ST_MEM(BPF_H, R10, -8, 12345), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_H, R0, R0, -8), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 12345 } }, + .stack_depth = 8, + }, + { + "LDX_MEM_W: operand register aliasing", + .u.insns_int = { + BPF_ST_MEM(BPF_W, R10, -8, 123456789), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_W, R0, R0, -8), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123456789 } }, + .stack_depth = 8, + }, + { + "LDX_MEM_DW: operand register aliasing", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x123456789abcdefULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_DW, R0, R0, -8), + BPF_ALU64_REG(BPF_SUB, R0, R1), + BPF_MOV64_REG(R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU64_REG(BPF_OR, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + /* + * Register (non-)clobbering tests for the case where a JIT implements + * complex ALU or ATOMIC operations via function calls. If so, the + * function call must be transparent to the eBPF registers. The JIT + * must therefore save and restore relevant registers across the call. + * The following tests check that the eBPF registers retain their + * values after such an operation. Mainly intended for complex ALU + * and atomic operation, but we run it for all. You never know... + * + * Note that each operations should be tested twice with different + * destinations, to check preservation for all registers. + */ +#define BPF_TEST_CLOBBER_ALU(alu, op, dst, src) \ + { \ + #alu "_" #op " to " #dst ": no clobbering", \ + .u.insns_int = { \ + BPF_ALU64_IMM(BPF_MOV, R0, R0), \ + BPF_ALU64_IMM(BPF_MOV, R1, R1), \ + BPF_ALU64_IMM(BPF_MOV, R2, R2), \ + BPF_ALU64_IMM(BPF_MOV, R3, R3), \ + BPF_ALU64_IMM(BPF_MOV, R4, R4), \ + BPF_ALU64_IMM(BPF_MOV, R5, R5), \ + BPF_ALU64_IMM(BPF_MOV, R6, R6), \ + BPF_ALU64_IMM(BPF_MOV, R7, R7), \ + BPF_ALU64_IMM(BPF_MOV, R8, R8), \ + BPF_ALU64_IMM(BPF_MOV, R9, R9), \ + BPF_##alu(BPF_ ##op, dst, src), \ + BPF_ALU32_IMM(BPF_MOV, dst, dst), \ + BPF_JMP_IMM(BPF_JNE, R0, R0, 10), \ + BPF_JMP_IMM(BPF_JNE, R1, R1, 9), \ + BPF_JMP_IMM(BPF_JNE, R2, R2, 8), \ + BPF_JMP_IMM(BPF_JNE, R3, R3, 7), \ + BPF_JMP_IMM(BPF_JNE, R4, R4, 6), \ + BPF_JMP_IMM(BPF_JNE, R5, R5, 5), \ + BPF_JMP_IMM(BPF_JNE, R6, R6, 4), \ + BPF_JMP_IMM(BPF_JNE, R7, R7, 3), \ + BPF_JMP_IMM(BPF_JNE, R8, R8, 2), \ + BPF_JMP_IMM(BPF_JNE, R9, R9, 1), \ + BPF_ALU64_IMM(BPF_MOV, R0, 1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 1 } } \ + } + /* ALU64 operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R9, 123456789), + /* ALU32 immediate operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R9, 123456789), + /* ALU64 register operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R9, R1), + /* ALU32 register operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R9, R1), +#undef BPF_TEST_CLOBBER_ALU +#define BPF_TEST_CLOBBER_ATOMIC(width, op) \ + { \ + "Atomic_" #width " " #op ": no clobbering", \ + .u.insns_int = { \ + BPF_ALU64_IMM(BPF_MOV, R0, 0), \ + BPF_ALU64_IMM(BPF_MOV, R1, 1), \ + BPF_ALU64_IMM(BPF_MOV, R2, 2), \ + BPF_ALU64_IMM(BPF_MOV, R3, 3), \ + BPF_ALU64_IMM(BPF_MOV, R4, 4), \ + BPF_ALU64_IMM(BPF_MOV, R5, 5), \ + BPF_ALU64_IMM(BPF_MOV, R6, 6), \ + BPF_ALU64_IMM(BPF_MOV, R7, 7), \ + BPF_ALU64_IMM(BPF_MOV, R8, 8), \ + BPF_ALU64_IMM(BPF_MOV, R9, 9), \ + BPF_ST_MEM(width, R10, -8, \ + (op) == BPF_CMPXCHG ? 0 : \ + (op) & BPF_FETCH ? 1 : 0), \ + BPF_ATOMIC_OP(width, op, R10, R1, -8), \ + BPF_JMP_IMM(BPF_JNE, R0, 0, 10), \ + BPF_JMP_IMM(BPF_JNE, R1, 1, 9), \ + BPF_JMP_IMM(BPF_JNE, R2, 2, 8), \ + BPF_JMP_IMM(BPF_JNE, R3, 3, 7), \ + BPF_JMP_IMM(BPF_JNE, R4, 4, 6), \ + BPF_JMP_IMM(BPF_JNE, R5, 5, 5), \ + BPF_JMP_IMM(BPF_JNE, R6, 6, 4), \ + BPF_JMP_IMM(BPF_JNE, R7, 7, 3), \ + BPF_JMP_IMM(BPF_JNE, R8, 8, 2), \ + BPF_JMP_IMM(BPF_JNE, R9, 9, 1), \ + BPF_ALU64_IMM(BPF_MOV, R0, 1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 1 } }, \ + .stack_depth = 8, \ + } + /* 64-bit atomic operations, register clobbering */ + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XCHG), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_CMPXCHG), + /* 32-bit atomic operations, register clobbering */ + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XCHG), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_CMPXCHG), +#undef BPF_TEST_CLOBBER_ATOMIC + /* Checking that ALU32 src is not zero extended in place */ +#define BPF_ALU32_SRC_ZEXT(op) \ + { \ + "ALU32_" #op "_X: src preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R1, 0x0123456789acbdefULL),\ + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL),\ + BPF_ALU64_REG(BPF_MOV, R0, R1), \ + BPF_ALU32_REG(BPF_##op, R2, R1), \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + } + BPF_ALU32_SRC_ZEXT(MOV), + BPF_ALU32_SRC_ZEXT(AND), + BPF_ALU32_SRC_ZEXT(OR), + BPF_ALU32_SRC_ZEXT(XOR), + BPF_ALU32_SRC_ZEXT(ADD), + BPF_ALU32_SRC_ZEXT(SUB), + BPF_ALU32_SRC_ZEXT(MUL), + BPF_ALU32_SRC_ZEXT(DIV), + BPF_ALU32_SRC_ZEXT(MOD), +#undef BPF_ALU32_SRC_ZEXT + /* Checking that ATOMIC32 src is not zero extended in place */ +#define BPF_ATOMIC32_SRC_ZEXT(op) \ + { \ + "ATOMIC_W_" #op ": src preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R0, 0x0123456789acbdefULL), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ST_MEM(BPF_W, R10, -4, 0), \ + BPF_ATOMIC_OP(BPF_W, BPF_##op, R10, R1, -4), \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + .stack_depth = 8, \ + } + BPF_ATOMIC32_SRC_ZEXT(ADD), + BPF_ATOMIC32_SRC_ZEXT(AND), + BPF_ATOMIC32_SRC_ZEXT(OR), + BPF_ATOMIC32_SRC_ZEXT(XOR), +#undef BPF_ATOMIC32_SRC_ZEXT + /* Checking that CMPXCHG32 src is not zero extended in place */ + { + "ATOMIC_W_CMPXCHG: src preserved in zext", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0123456789acbdefULL), + BPF_ALU64_REG(BPF_MOV, R2, R1), + BPF_ALU64_REG(BPF_MOV, R0, 0), + BPF_ST_MEM(BPF_W, R10, -4, 0), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R1, -4), + BPF_ALU64_REG(BPF_SUB, R1, R2), + BPF_ALU64_REG(BPF_MOV, R2, R1), + BPF_ALU64_IMM(BPF_RSH, R2, 32), + BPF_ALU64_REG(BPF_OR, R1, R2), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + /* Checking that JMP32 immediate src is not zero extended in place */ +#define BPF_JMP32_IMM_ZEXT(op) \ + { \ + "JMP32_" #op "_K: operand preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_JMP32_IMM(BPF_##op, R0, 1234, 1), \ + BPF_JMP_A(0), /* Nop */ \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + } + BPF_JMP32_IMM_ZEXT(JEQ), + BPF_JMP32_IMM_ZEXT(JNE), + BPF_JMP32_IMM_ZEXT(JSET), + BPF_JMP32_IMM_ZEXT(JGT), + BPF_JMP32_IMM_ZEXT(JGE), + BPF_JMP32_IMM_ZEXT(JLT), + BPF_JMP32_IMM_ZEXT(JLE), + BPF_JMP32_IMM_ZEXT(JSGT), + BPF_JMP32_IMM_ZEXT(JSGE), + BPF_JMP32_IMM_ZEXT(JSGT), + BPF_JMP32_IMM_ZEXT(JSLT), + BPF_JMP32_IMM_ZEXT(JSLE), +#undef BPF_JMP2_IMM_ZEXT + /* Checking that JMP32 dst & src are not zero extended in place */ +#define BPF_JMP32_REG_ZEXT(op) \ + { \ + "JMP32_" #op "_X: operands preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ + BPF_LD_IMM64(R1, 0xfedcba9876543210ULL),\ + BPF_ALU64_REG(BPF_MOV, R2, R0), \ + BPF_ALU64_REG(BPF_MOV, R3, R1), \ + BPF_JMP32_IMM(BPF_##op, R0, R1, 1), \ + BPF_JMP_A(0), /* Nop */ \ + BPF_ALU64_REG(BPF_SUB, R0, R2), \ + BPF_ALU64_REG(BPF_SUB, R1, R3), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + } + BPF_JMP32_REG_ZEXT(JEQ), + BPF_JMP32_REG_ZEXT(JNE), + BPF_JMP32_REG_ZEXT(JSET), + BPF_JMP32_REG_ZEXT(JGT), + BPF_JMP32_REG_ZEXT(JGE), + BPF_JMP32_REG_ZEXT(JLT), + BPF_JMP32_REG_ZEXT(JLE), + BPF_JMP32_REG_ZEXT(JSGT), + BPF_JMP32_REG_ZEXT(JSGE), + BPF_JMP32_REG_ZEXT(JSGT), + BPF_JMP32_REG_ZEXT(JSLT), + BPF_JMP32_REG_ZEXT(JSLE), +#undef BPF_JMP2_REG_ZEXT + /* ALU64 K register combinations */ + { + "ALU64_MOV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_imm_regs, + }, + { + "ALU64_AND_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_imm_regs, + }, + { + "ALU64_OR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_imm_regs, + }, + { + "ALU64_XOR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_imm_regs, + }, + { + "ALU64_LSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_imm_regs, + }, + { + "ALU64_RSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_imm_regs, + }, + { + "ALU64_ARSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_imm_regs, + }, + { + "ALU64_ADD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_imm_regs, + }, + { + "ALU64_SUB_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_imm_regs, + }, + { + "ALU64_MUL_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_imm_regs, + }, + { + "ALU64_DIV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_imm_regs, + }, + { + "ALU64_MOD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_imm_regs, + }, + /* ALU32 K registers */ + { + "ALU32_MOV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_imm_regs, + }, + { + "ALU32_AND_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_imm_regs, + }, + { + "ALU32_OR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_imm_regs, + }, + { + "ALU32_XOR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_imm_regs, + }, + { + "ALU32_LSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_imm_regs, + }, + { + "ALU32_RSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_imm_regs, + }, + { + "ALU32_ARSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_imm_regs, + }, + { + "ALU32_ADD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_imm_regs, + }, + { + "ALU32_SUB_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_imm_regs, + }, + { + "ALU32_MUL_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_imm_regs, + }, + { + "ALU32_DIV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_imm_regs, + }, + { + "ALU32_MOD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_imm_regs, + }, + /* ALU64 X register combinations */ + { + "ALU64_MOV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_reg_pairs, + }, + { + "ALU64_AND_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_reg_pairs, + }, + { + "ALU64_OR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_reg_pairs, + }, + { + "ALU64_XOR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_reg_pairs, + }, + { + "ALU64_LSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_reg_pairs, + }, + { + "ALU64_RSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_reg_pairs, + }, + { + "ALU64_ARSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_reg_pairs, + }, + { + "ALU64_ADD_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_reg_pairs, + }, + { + "ALU64_SUB_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_reg_pairs, + }, + { + "ALU64_MUL_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_reg_pairs, + }, + { + "ALU64_DIV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_reg_pairs, + }, + { + "ALU64_MOD_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_reg_pairs, + }, + /* ALU32 X register combinations */ + { + "ALU32_MOV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_reg_pairs, + }, + { + "ALU32_AND_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_reg_pairs, + }, + { + "ALU32_OR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_reg_pairs, + }, + { + "ALU32_XOR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_reg_pairs, + }, + { + "ALU32_LSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_reg_pairs, + }, + { + "ALU32_RSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_reg_pairs, + }, + { + "ALU32_ARSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_reg_pairs, + }, + { + "ALU32_ADD_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_reg_pairs, + }, + { + "ALU32_SUB_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_reg_pairs, + }, + { + "ALU32_MUL_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_reg_pairs, + }, + { + "ALU32_DIV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_reg_pairs, + }, + { + "ALU32_MOD_X register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_reg_pairs, + }, + /* Exhaustive test of ALU64 shift operations */ + { + "ALU64_LSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_imm, + }, + { + "ALU64_RSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_imm, + }, + { + "ALU64_ARSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_imm, + }, + { + "ALU64_LSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_reg, + }, + { + "ALU64_RSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_reg, + }, + { + "ALU64_ARSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_reg, + }, + /* Exhaustive test of ALU32 shift operations */ + { + "ALU32_LSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_imm, + }, + { + "ALU32_RSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_imm, + }, + { + "ALU32_ARSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_imm, + }, + { + "ALU32_LSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_reg, + }, + { + "ALU32_RSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_reg, + }, + { + "ALU32_ARSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_reg, + }, + /* + * Exhaustive test of ALU64 shift operations when + * source and destination register are the same. + */ + { + "ALU64_LSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_same_reg, + }, + { + "ALU64_RSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_same_reg, + }, + { + "ALU64_ARSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_same_reg, + }, + /* + * Exhaustive test of ALU32 shift operations when + * source and destination register are the same. + */ + { + "ALU32_LSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_same_reg, + }, + { + "ALU32_RSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_same_reg, + }, + { + "ALU32_ARSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_same_reg, + }, + /* ALU64 immediate magnitudes */ + { + "ALU64_MOV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_AND_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_OR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_XOR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_ADD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_SUB_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MUL_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_DIV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MOD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* ALU32 immediate magnitudes */ + { + "ALU32_MOV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_AND_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_OR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_XOR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_ADD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_SUB_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MUL_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_DIV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MOD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* ALU64 register magnitudes */ + { + "ALU64_MOV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_AND_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_OR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_XOR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_ADD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_SUB_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MUL_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_DIV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MOD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* ALU32 register magnitudes */ + { + "ALU32_MOV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_AND_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_OR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_XOR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_ADD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_SUB_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MUL_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_DIV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MOD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* LD_IMM64 immediate magnitudes and byte patterns */ + { + "LD_IMM64: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_magn, + }, + { + "LD_IMM64: checker byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_checker, + }, + { + "LD_IMM64: random positive and zero byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_pos_zero, + }, + { + "LD_IMM64: random negative and zero byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_neg_zero, + }, + { + "LD_IMM64: random positive and negative byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_pos_neg, + }, + /* 64-bit ATOMIC register combinations */ + { + "ATOMIC_DW_ADD: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_AND: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_OR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_XOR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_ADD_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_AND_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_OR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_XOR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_XCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xchg_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_CMPXCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_cmpxchg_reg_pairs, + .stack_depth = 8, + }, + /* 32-bit ATOMIC register combinations */ + { + "ATOMIC_W_ADD: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_AND: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_OR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_XOR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_ADD_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_AND_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_OR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_XOR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_XCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xchg_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_CMPXCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_cmpxchg_reg_pairs, + .stack_depth = 8, + }, + /* 64-bit ATOMIC magnitudes */ + { + "ATOMIC_DW_ADD: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_AND: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_OR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_XOR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_ADD_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_AND_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_OR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_XOR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_XCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xchg, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_CMPXCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_cmpxchg64, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* 64-bit atomic magnitudes */ + { + "ATOMIC_W_ADD: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_AND: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_OR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_XOR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_ADD_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_AND_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_OR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_XOR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_XCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xchg, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_CMPXCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_cmpxchg32, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP immediate magnitudes */ + { + "JMP_JSET_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jset_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JEQ_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jeq_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JNE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jne_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jlt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jslt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP register magnitudes */ + { + "JMP_JSET_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jset_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JEQ_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jeq_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JNE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jne_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jlt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jslt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP32 immediate magnitudes */ + { + "JMP32_JSET_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jset_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JEQ_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jeq_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JNE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jne_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jlt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jslt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP32 register magnitudes */ + { + "JMP32_JSET_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jset_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JEQ_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jeq_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JNE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jne_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jlt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jslt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* Conditional jumps with constant decision */ + { + "JMP_JSET_K: imm = 0 -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JSET, R1, 0, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JLT_K: imm = 0 -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JLT, R1, 0, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JGE_K: imm = 0 -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JGE, R1, 0, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JGT_K: imm = 0xffffffff -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JGT, R1, U32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JLE_K: imm = 0xffffffff -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JLE, R1, U32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP32_JSGT_K: imm = 0x7fffffff -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSGT, R1, S32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP32_JSGE_K: imm = -0x80000000 -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSGE, R1, S32_MIN, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP32_JSLT_K: imm = -0x80000000 -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSLT, R1, S32_MIN, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP32_JSLE_K: imm = 0x7fffffff -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSLE, R1, S32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JEQ_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JEQ, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JGE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JGE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JLE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JLE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JSGE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSGE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JSLE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSLE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JNE_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JNE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JGT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JGT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JLT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JLT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JSGT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSGT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JSLT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSLT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + /* Short relative jumps */ + { + "Short relative jump: offset=0", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 0), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=1", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=2", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=3", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 3), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=4", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 4), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + /* Conditional branch conversions */ + { + "Long conditional jump: taken at runtime", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_max_jmp_taken, + }, + { + "Long conditional jump: not taken at runtime", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 2 } }, + .fill_helper = bpf_fill_max_jmp_not_taken, + }, + { + "Long conditional jump: always taken, known at JIT time", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_max_jmp_always_taken, + }, + { + "Long conditional jump: never taken, known at JIT time", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 2 } }, + .fill_helper = bpf_fill_max_jmp_never_taken, + }, + /* Staggered jump sequences, immediate */ + { + "Staggered jumps: JMP_JA", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_ja, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JEQ_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JNE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSET_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + /* Staggered jump sequences, register */ + { + "Staggered jumps: JMP_JEQ_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JNE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSET_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + /* Staggered jump sequences, JMP32 immediate */ + { + "Staggered jumps: JMP32_JEQ_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JNE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSET_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + /* Staggered jump sequences, JMP32 register */ + { + "Staggered jumps: JMP32_JEQ_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JNE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSET_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, }; static struct net_device dev; @@ -6624,6 +14213,8 @@ static struct bpf_prog *generate_filter(int which, int *err) fp->type = BPF_PROG_TYPE_SOCKET_FILTER; memcpy(fp->insnsi, fptr, fp->len * sizeof(struct bpf_insn)); fp->aux->stack_depth = tests[which].stack_depth; + fp->aux->verifier_zext = !!(tests[which].aux & + FLAG_VERIFIER_ZEXT); /* We cannot error here as we don't need type compatibility * checks. @@ -6660,14 +14251,14 @@ static int __run_one(const struct bpf_prog *fp, const void *data, u64 start, finish; int ret = 0, i; - preempt_disable(); + migrate_disable(); start = ktime_get_ns(); for (i = 0; i < runs; i++) - ret = BPF_PROG_RUN(fp, data); + ret = bpf_prog_run(fp, data); finish = ktime_get_ns(); - preempt_enable(); + migrate_enable(); *duration = finish - start; do_div(*duration, runs); @@ -6679,12 +14270,22 @@ static int run_one(const struct bpf_prog *fp, struct bpf_test *test) { int err_cnt = 0, i, runs = MAX_TESTRUNS; + if (test->nr_testruns) + runs = min(test->nr_testruns, MAX_TESTRUNS); + for (i = 0; i < MAX_SUBTESTS; i++) { void *data; u64 duration; u32 ret; - if (test->test[i].data_size == 0 && + /* + * NOTE: Several sub-tests may be present, in which case + * a zero {data_size, result} tuple indicates the end of + * the sub-test array. The first test is always run, + * even if both data_size and result happen to be zero. + */ + if (i > 0 && + test->test[i].data_size == 0 && test->test[i].result == 0) break; @@ -6715,86 +14316,9 @@ module_param_string(test_name, test_name, sizeof(test_name), 0); static int test_id = -1; module_param(test_id, int, 0); -static int test_range[2] = { 0, ARRAY_SIZE(tests) - 1 }; +static int test_range[2] = { 0, INT_MAX }; module_param_array(test_range, int, NULL, 0); -static __init int find_test_index(const char *test_name) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (!strcmp(tests[i].descr, test_name)) - return i; - } - return -1; -} - -static __init int prepare_bpf_tests(void) -{ - int i; - - if (test_id >= 0) { - /* - * if a test_id was specified, use test_range to - * cover only that test. - */ - if (test_id >= ARRAY_SIZE(tests)) { - pr_err("test_bpf: invalid test_id specified.\n"); - return -EINVAL; - } - - test_range[0] = test_id; - test_range[1] = test_id; - } else if (*test_name) { - /* - * if a test_name was specified, find it and setup - * test_range to cover only that test. - */ - int idx = find_test_index(test_name); - - if (idx < 0) { - pr_err("test_bpf: no test named '%s' found.\n", - test_name); - return -EINVAL; - } - test_range[0] = idx; - test_range[1] = idx; - } else { - /* - * check that the supplied test_range is valid. - */ - if (test_range[0] >= ARRAY_SIZE(tests) || - test_range[1] >= ARRAY_SIZE(tests) || - test_range[0] < 0 || test_range[1] < 0) { - pr_err("test_bpf: test_range is out of bound.\n"); - return -EINVAL; - } - - if (test_range[1] < test_range[0]) { - pr_err("test_bpf: test_range is ending before it starts.\n"); - return -EINVAL; - } - } - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (tests[i].fill_helper && - tests[i].fill_helper(&tests[i]) < 0) - return -ENOMEM; - } - - return 0; -} - -static __init void destroy_bpf_tests(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (tests[i].fill_helper) - kfree(tests[i].u.ptr.insns); - } -} - static bool exclude_test(int test_id) { return test_id < test_range[0] || test_id > test_range[1]; @@ -6841,6 +14365,7 @@ static __init struct sk_buff *build_test_skb(void) skb_shinfo(skb[0])->gso_type |= SKB_GSO_DODGY; skb_shinfo(skb[0])->gso_segs = 0; skb_shinfo(skb[0])->frag_list = skb[1]; + skb_shinfo(skb[0])->hwtstamps.hwtstamp = 1000; /* adjust skb[0]'s len */ skb[0]->len += skb[1]->len; @@ -6965,6 +14490,10 @@ static __init int test_skb_segment(void) for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { const struct skb_segment_test *test = &skb_segment_tests[i]; + cond_resched(); + if (exclude_test(i)) + continue; + pr_info("#%d %s ", i, test->descr); if (test_skb_segment_single(test)) { @@ -6996,7 +14525,19 @@ static __init int test_bpf(void) pr_info("#%d %s ", i, tests[i].descr); + if (tests[i].fill_helper && + tests[i].fill_helper(&tests[i]) < 0) { + pr_cont("FAIL to prog_fill\n"); + continue; + } + fp = generate_filter(i, &err); + + if (tests[i].fill_helper) { + kfree(tests[i].u.ptr.insns); + tests[i].u.ptr.insns = NULL; + } + if (fp == NULL) { if (err == 0) { pass_cnt++; @@ -7030,20 +14571,482 @@ static __init int test_bpf(void) return err_cnt ? -EINVAL : 0; } +struct tail_call_test { + const char *descr; + struct bpf_insn insns[MAX_INSNS]; + int flags; + int result; + int stack_depth; +}; + +/* Flags that can be passed to tail call test cases */ +#define FLAG_NEED_STATE BIT(0) +#define FLAG_RESULT_IN_STATE BIT(1) + +/* + * Magic marker used in test snippets for tail calls below. + * BPF_LD/MOV to R2 and R2 with this immediate value is replaced + * with the proper values by the test runner. + */ +#define TAIL_CALL_MARKER 0x7a11ca11 + +/* Special offset to indicate a NULL call target */ +#define TAIL_CALL_NULL 0x7fff + +/* Special offset to indicate an out-of-range index */ +#define TAIL_CALL_INVALID 0x7ffe + +#define TAIL_CALL(offset) \ + BPF_LD_IMM64(R2, TAIL_CALL_MARKER), \ + BPF_RAW_INSN(BPF_ALU | BPF_MOV | BPF_K, R3, 0, \ + offset, TAIL_CALL_MARKER), \ + BPF_JMP_IMM(BPF_TAIL_CALL, 0, 0, 0) + +/* + * A test function to be called from a BPF program, clobbering a lot of + * CPU registers in the process. A JITed BPF program calling this function + * must save and restore any caller-saved registers it uses for internal + * state, for example the current tail call count. + */ +BPF_CALL_1(bpf_test_func, u64, arg) +{ + char buf[64]; + long a = 0; + long b = 1; + long c = 2; + long d = 3; + long e = 4; + long f = 5; + long g = 6; + long h = 7; + + return snprintf(buf, sizeof(buf), + "%ld %lu %lx %ld %lu %lx %ld %lu %x", + a, b, c, d, e, f, g, h, (int)arg); +} +#define BPF_FUNC_test_func __BPF_FUNC_MAX_ID + +/* + * Tail call tests. Each test case may call any other test in the table, + * including itself, specified as a relative index offset from the calling + * test. The index TAIL_CALL_NULL can be used to specify a NULL target + * function to test the JIT error path. Similarly, the index TAIL_CALL_INVALID + * results in a target index that is out of range. + */ +static struct tail_call_test tail_call_tests[] = { + { + "Tail call leaf", + .insns = { + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_ALU64_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + }, + .result = 1, + }, + { + "Tail call 2", + .insns = { + BPF_ALU64_IMM(BPF_ADD, R1, 2), + TAIL_CALL(-1), + BPF_ALU64_IMM(BPF_MOV, R0, -1), + BPF_EXIT_INSN(), + }, + .result = 3, + }, + { + "Tail call 3", + .insns = { + BPF_ALU64_IMM(BPF_ADD, R1, 3), + TAIL_CALL(-1), + BPF_ALU64_IMM(BPF_MOV, R0, -1), + BPF_EXIT_INSN(), + }, + .result = 6, + }, + { + "Tail call 4", + .insns = { + BPF_ALU64_IMM(BPF_ADD, R1, 4), + TAIL_CALL(-1), + BPF_ALU64_IMM(BPF_MOV, R0, -1), + BPF_EXIT_INSN(), + }, + .result = 10, + }, + { + "Tail call error path, max count reached", + .insns = { + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), + TAIL_CALL(0), + BPF_EXIT_INSN(), + }, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = (MAX_TAIL_CALL_CNT + 1 + 1) * MAX_TESTRUNS, + }, + { + "Tail call count preserved across function calls", + .insns = { + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), + BPF_STX_MEM(BPF_DW, R10, R1, -8), + BPF_CALL_REL(BPF_FUNC_get_numa_node_id), + BPF_CALL_REL(BPF_FUNC_ktime_get_ns), + BPF_CALL_REL(BPF_FUNC_ktime_get_boot_ns), + BPF_CALL_REL(BPF_FUNC_ktime_get_coarse_ns), + BPF_CALL_REL(BPF_FUNC_jiffies64), + BPF_CALL_REL(BPF_FUNC_test_func), + BPF_LDX_MEM(BPF_DW, R1, R10, -8), + BPF_ALU32_REG(BPF_MOV, R0, R1), + TAIL_CALL(0), + BPF_EXIT_INSN(), + }, + .stack_depth = 8, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = (MAX_TAIL_CALL_CNT + 1 + 1) * MAX_TESTRUNS, + }, + { + "Tail call error path, NULL target", + .insns = { + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), + TAIL_CALL(TAIL_CALL_NULL), + BPF_EXIT_INSN(), + }, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = MAX_TESTRUNS, + }, + { + "Tail call error path, index out of range", + .insns = { + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), + TAIL_CALL(TAIL_CALL_INVALID), + BPF_EXIT_INSN(), + }, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = MAX_TESTRUNS, + }, +}; + +static void __init destroy_tail_call_tests(struct bpf_array *progs) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) + if (progs->ptrs[i]) + bpf_prog_free(progs->ptrs[i]); + kfree(progs); +} + +static __init int prepare_tail_call_tests(struct bpf_array **pprogs) +{ + int ntests = ARRAY_SIZE(tail_call_tests); + struct bpf_array *progs; + int which, err; + + /* Allocate the table of programs to be used for tall calls */ + progs = kzalloc(sizeof(*progs) + (ntests + 1) * sizeof(progs->ptrs[0]), + GFP_KERNEL); + if (!progs) + goto out_nomem; + + /* Create all eBPF programs and populate the table */ + for (which = 0; which < ntests; which++) { + struct tail_call_test *test = &tail_call_tests[which]; + struct bpf_prog *fp; + int len, i; + + /* Compute the number of program instructions */ + for (len = 0; len < MAX_INSNS; len++) { + struct bpf_insn *insn = &test->insns[len]; + + if (len < MAX_INSNS - 1 && + insn->code == (BPF_LD | BPF_DW | BPF_IMM)) + len++; + if (insn->code == 0) + break; + } + + /* Allocate and initialize the program */ + fp = bpf_prog_alloc(bpf_prog_size(len), 0); + if (!fp) + goto out_nomem; + + fp->len = len; + fp->type = BPF_PROG_TYPE_SOCKET_FILTER; + fp->aux->stack_depth = test->stack_depth; + memcpy(fp->insnsi, test->insns, len * sizeof(struct bpf_insn)); + + /* Relocate runtime tail call offsets and addresses */ + for (i = 0; i < len; i++) { + struct bpf_insn *insn = &fp->insnsi[i]; + long addr = 0; + + switch (insn->code) { + case BPF_LD | BPF_DW | BPF_IMM: + if (insn->imm != TAIL_CALL_MARKER) + break; + insn[0].imm = (u32)(long)progs; + insn[1].imm = ((u64)(long)progs) >> 32; + break; + + case BPF_ALU | BPF_MOV | BPF_K: + if (insn->imm != TAIL_CALL_MARKER) + break; + if (insn->off == TAIL_CALL_NULL) + insn->imm = ntests; + else if (insn->off == TAIL_CALL_INVALID) + insn->imm = ntests + 1; + else + insn->imm = which + insn->off; + insn->off = 0; + break; + + case BPF_JMP | BPF_CALL: + if (insn->src_reg != BPF_PSEUDO_CALL) + break; + switch (insn->imm) { + case BPF_FUNC_get_numa_node_id: + addr = (long)&numa_node_id; + break; + case BPF_FUNC_ktime_get_ns: + addr = (long)&ktime_get_ns; + break; + case BPF_FUNC_ktime_get_boot_ns: + addr = (long)&ktime_get_boot_fast_ns; + break; + case BPF_FUNC_ktime_get_coarse_ns: + addr = (long)&ktime_get_coarse_ns; + break; + case BPF_FUNC_jiffies64: + addr = (long)&get_jiffies_64; + break; + case BPF_FUNC_test_func: + addr = (long)&bpf_test_func; + break; + default: + err = -EFAULT; + goto out_err; + } + *insn = BPF_EMIT_CALL(addr); + if ((long)__bpf_call_base + insn->imm != addr) + *insn = BPF_JMP_A(0); /* Skip: NOP */ + break; + } + } + + fp = bpf_prog_select_runtime(fp, &err); + if (err) + goto out_err; + + progs->ptrs[which] = fp; + } + + /* The last entry contains a NULL program pointer */ + progs->map.max_entries = ntests + 1; + *pprogs = progs; + return 0; + +out_nomem: + err = -ENOMEM; + +out_err: + if (progs) + destroy_tail_call_tests(progs); + return err; +} + +static __init int test_tail_calls(struct bpf_array *progs) +{ + int i, err_cnt = 0, pass_cnt = 0; + int jit_cnt = 0, run_cnt = 0; + + for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { + struct tail_call_test *test = &tail_call_tests[i]; + struct bpf_prog *fp = progs->ptrs[i]; + int *data = NULL; + int state = 0; + u64 duration; + int ret; + + cond_resched(); + if (exclude_test(i)) + continue; + + pr_info("#%d %s ", i, test->descr); + if (!fp) { + err_cnt++; + continue; + } + pr_cont("jited:%u ", fp->jited); + + run_cnt++; + if (fp->jited) + jit_cnt++; + + if (test->flags & FLAG_NEED_STATE) + data = &state; + ret = __run_one(fp, data, MAX_TESTRUNS, &duration); + if (test->flags & FLAG_RESULT_IN_STATE) + ret = state; + if (ret == test->result) { + pr_cont("%lld PASS", duration); + pass_cnt++; + } else { + pr_cont("ret %d != %d FAIL", ret, test->result); + err_cnt++; + } + } + + pr_info("%s: Summary: %d PASSED, %d FAILED, [%d/%d JIT'ed]\n", + __func__, pass_cnt, err_cnt, jit_cnt, run_cnt); + + return err_cnt ? -EINVAL : 0; +} + +static char test_suite[32]; +module_param_string(test_suite, test_suite, sizeof(test_suite), 0); + +static __init int find_test_index(const char *test_name) +{ + int i; + + if (!strcmp(test_suite, "test_bpf")) { + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (!strcmp(tests[i].descr, test_name)) + return i; + } + } + + if (!strcmp(test_suite, "test_tail_calls")) { + for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { + if (!strcmp(tail_call_tests[i].descr, test_name)) + return i; + } + } + + if (!strcmp(test_suite, "test_skb_segment")) { + for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { + if (!strcmp(skb_segment_tests[i].descr, test_name)) + return i; + } + } + + return -1; +} + +static __init int prepare_test_range(void) +{ + int valid_range; + + if (!strcmp(test_suite, "test_bpf")) + valid_range = ARRAY_SIZE(tests); + else if (!strcmp(test_suite, "test_tail_calls")) + valid_range = ARRAY_SIZE(tail_call_tests); + else if (!strcmp(test_suite, "test_skb_segment")) + valid_range = ARRAY_SIZE(skb_segment_tests); + else + return 0; + + if (test_id >= 0) { + /* + * if a test_id was specified, use test_range to + * cover only that test. + */ + if (test_id >= valid_range) { + pr_err("test_bpf: invalid test_id specified for '%s' suite.\n", + test_suite); + return -EINVAL; + } + + test_range[0] = test_id; + test_range[1] = test_id; + } else if (*test_name) { + /* + * if a test_name was specified, find it and setup + * test_range to cover only that test. + */ + int idx = find_test_index(test_name); + + if (idx < 0) { + pr_err("test_bpf: no test named '%s' found for '%s' suite.\n", + test_name, test_suite); + return -EINVAL; + } + test_range[0] = idx; + test_range[1] = idx; + } else if (test_range[0] != 0 || test_range[1] != INT_MAX) { + /* + * check that the supplied test_range is valid. + */ + if (test_range[0] < 0 || test_range[1] >= valid_range) { + pr_err("test_bpf: test_range is out of bound for '%s' suite.\n", + test_suite); + return -EINVAL; + } + + if (test_range[1] < test_range[0]) { + pr_err("test_bpf: test_range is ending before it starts.\n"); + return -EINVAL; + } + } + + return 0; +} + static int __init test_bpf_init(void) { + struct bpf_array *progs = NULL; int ret; - ret = prepare_bpf_tests(); + if (strlen(test_suite) && + strcmp(test_suite, "test_bpf") && + strcmp(test_suite, "test_tail_calls") && + strcmp(test_suite, "test_skb_segment")) { + pr_err("test_bpf: invalid test_suite '%s' specified.\n", test_suite); + return -EINVAL; + } + + /* + * if test_suite is not specified, but test_id, test_name or test_range + * is specified, set 'test_bpf' as the default test suite. + */ + if (!strlen(test_suite) && + (test_id != -1 || strlen(test_name) || + (test_range[0] != 0 || test_range[1] != INT_MAX))) { + pr_info("test_bpf: set 'test_bpf' as the default test_suite.\n"); + strscpy(test_suite, "test_bpf", sizeof(test_suite)); + } + + ret = prepare_test_range(); if (ret < 0) return ret; - ret = test_bpf(); - destroy_bpf_tests(); - if (ret) - return ret; + if (!strlen(test_suite) || !strcmp(test_suite, "test_bpf")) { + ret = test_bpf(); + if (ret) + return ret; + } - return test_skb_segment(); + if (!strlen(test_suite) || !strcmp(test_suite, "test_tail_calls")) { + ret = prepare_tail_call_tests(&progs); + if (ret) + return ret; + ret = test_tail_calls(progs); + destroy_tail_call_tests(progs); + if (ret) + return ret; + } + + if (!strlen(test_suite) || !strcmp(test_suite, "test_skb_segment")) + return test_skb_segment(); + + return 0; } static void __exit test_bpf_exit(void) diff --git a/lib/test_firmware.c b/lib/test_firmware.c index 251213c872b5..1bccd6cd5f48 100644 --- a/lib/test_firmware.c +++ b/lib/test_firmware.c @@ -24,6 +24,9 @@ #include <linux/delay.h> #include <linux/kthread.h> #include <linux/vmalloc.h> +#include <linux/efi_embedded_fw.h> + +MODULE_IMPORT_NS(TEST_FIRMWARE); #define TEST_FIRMWARE_NAME "test-firmware.bin" #define TEST_FIRMWARE_NUM_REQS 4 @@ -49,6 +52,9 @@ struct test_batched_req { * @name: the name of the firmware file to look for * @into_buf: when the into_buf is used if this is true * request_firmware_into_buf() will be used instead. + * @buf_size: size of buf to allocate when into_buf is true + * @file_offset: file offset to request when calling request_firmware_into_buf + * @partial: partial read opt when calling request_firmware_into_buf * @sync_direct: when the sync trigger is used if this is true * request_firmware_direct() will be used instead. * @send_uevent: whether or not to send a uevent for async requests @@ -88,6 +94,9 @@ struct test_batched_req { struct test_config { char *name; bool into_buf; + size_t buf_size; + size_t file_offset; + bool partial; bool sync_direct; bool send_uevent; u8 num_requests; @@ -182,6 +191,9 @@ static int __test_firmware_config_init(void) test_fw_config->num_requests = TEST_FIRMWARE_NUM_REQS; test_fw_config->send_uevent = true; test_fw_config->into_buf = false; + test_fw_config->buf_size = TEST_FIRMWARE_BUF_SIZE; + test_fw_config->file_offset = 0; + test_fw_config->partial = false; test_fw_config->sync_direct = false; test_fw_config->req_firmware = request_firmware; test_fw_config->test_result = 0; @@ -235,28 +247,35 @@ static ssize_t config_show(struct device *dev, dev_name(dev)); if (test_fw_config->name) - len += scnprintf(buf+len, PAGE_SIZE - len, + len += scnprintf(buf + len, PAGE_SIZE - len, "name:\t%s\n", test_fw_config->name); else - len += scnprintf(buf+len, PAGE_SIZE - len, + len += scnprintf(buf + len, PAGE_SIZE - len, "name:\tEMTPY\n"); - len += scnprintf(buf+len, PAGE_SIZE - len, + len += scnprintf(buf + len, PAGE_SIZE - len, "num_requests:\t%u\n", test_fw_config->num_requests); - len += scnprintf(buf+len, PAGE_SIZE - len, + len += scnprintf(buf + len, PAGE_SIZE - len, "send_uevent:\t\t%s\n", test_fw_config->send_uevent ? - "FW_ACTION_HOTPLUG" : - "FW_ACTION_NOHOTPLUG"); - len += scnprintf(buf+len, PAGE_SIZE - len, + "FW_ACTION_UEVENT" : + "FW_ACTION_NOUEVENT"); + len += scnprintf(buf + len, PAGE_SIZE - len, "into_buf:\t\t%s\n", test_fw_config->into_buf ? "true" : "false"); - len += scnprintf(buf+len, PAGE_SIZE - len, + len += scnprintf(buf + len, PAGE_SIZE - len, + "buf_size:\t%zu\n", test_fw_config->buf_size); + len += scnprintf(buf + len, PAGE_SIZE - len, + "file_offset:\t%zu\n", test_fw_config->file_offset); + len += scnprintf(buf + len, PAGE_SIZE - len, + "partial:\t\t%s\n", + test_fw_config->partial ? "true" : "false"); + len += scnprintf(buf + len, PAGE_SIZE - len, "sync_direct:\t\t%s\n", test_fw_config->sync_direct ? "true" : "false"); - len += scnprintf(buf+len, PAGE_SIZE - len, + len += scnprintf(buf + len, PAGE_SIZE - len, "read_fw_idx:\t%u\n", test_fw_config->read_fw_idx); mutex_unlock(&test_fw_mutex); @@ -309,58 +328,59 @@ static int test_dev_config_update_bool(const char *buf, size_t size, return ret; } -static ssize_t -test_dev_config_show_bool(char *buf, - bool config) +static ssize_t test_dev_config_show_bool(char *buf, bool val) { - bool val; - - mutex_lock(&test_fw_mutex); - val = config; - mutex_unlock(&test_fw_mutex); - return snprintf(buf, PAGE_SIZE, "%d\n", val); } -static ssize_t test_dev_config_show_int(char *buf, int cfg) +static int test_dev_config_update_size_t(const char *buf, + size_t size, + size_t *cfg) { - int val; + int ret; + long new; + + ret = kstrtol(buf, 10, &new); + if (ret) + return ret; mutex_lock(&test_fw_mutex); - val = cfg; + *(size_t *)cfg = new; mutex_unlock(&test_fw_mutex); + /* Always return full write size even if we didn't consume all */ + return size; +} + +static ssize_t test_dev_config_show_size_t(char *buf, size_t val) +{ + return snprintf(buf, PAGE_SIZE, "%zu\n", val); +} + +static ssize_t test_dev_config_show_int(char *buf, int val) +{ return snprintf(buf, PAGE_SIZE, "%d\n", val); } static int test_dev_config_update_u8(const char *buf, size_t size, u8 *cfg) { + u8 val; int ret; - long new; - ret = kstrtol(buf, 10, &new); + ret = kstrtou8(buf, 10, &val); if (ret) return ret; - if (new > U8_MAX) - return -EINVAL; - mutex_lock(&test_fw_mutex); - *(u8 *)cfg = new; + *(u8 *)cfg = val; mutex_unlock(&test_fw_mutex); /* Always return full write size even if we didn't consume all */ return size; } -static ssize_t test_dev_config_show_u8(char *buf, u8 cfg) +static ssize_t test_dev_config_show_u8(char *buf, u8 val) { - u8 val; - - mutex_lock(&test_fw_mutex); - val = cfg; - mutex_unlock(&test_fw_mutex); - return snprintf(buf, PAGE_SIZE, "%u\n", val); } @@ -419,6 +439,83 @@ static ssize_t config_into_buf_show(struct device *dev, } static DEVICE_ATTR_RW(config_into_buf); +static ssize_t config_buf_size_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int rc; + + mutex_lock(&test_fw_mutex); + if (test_fw_config->reqs) { + pr_err("Must call release_all_firmware prior to changing config\n"); + rc = -EINVAL; + mutex_unlock(&test_fw_mutex); + goto out; + } + mutex_unlock(&test_fw_mutex); + + rc = test_dev_config_update_size_t(buf, count, + &test_fw_config->buf_size); + +out: + return rc; +} + +static ssize_t config_buf_size_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return test_dev_config_show_size_t(buf, test_fw_config->buf_size); +} +static DEVICE_ATTR_RW(config_buf_size); + +static ssize_t config_file_offset_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int rc; + + mutex_lock(&test_fw_mutex); + if (test_fw_config->reqs) { + pr_err("Must call release_all_firmware prior to changing config\n"); + rc = -EINVAL; + mutex_unlock(&test_fw_mutex); + goto out; + } + mutex_unlock(&test_fw_mutex); + + rc = test_dev_config_update_size_t(buf, count, + &test_fw_config->file_offset); + +out: + return rc; +} + +static ssize_t config_file_offset_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return test_dev_config_show_size_t(buf, test_fw_config->file_offset); +} +static DEVICE_ATTR_RW(config_file_offset); + +static ssize_t config_partial_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + return test_dev_config_update_bool(buf, + count, + &test_fw_config->partial); +} + +static ssize_t config_partial_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return test_dev_config_show_bool(buf, test_fw_config->partial); +} +static DEVICE_ATTR_RW(config_partial); + static ssize_t config_sync_direct_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -507,6 +604,64 @@ out: } static DEVICE_ATTR_WO(trigger_request); +#ifdef CONFIG_EFI_EMBEDDED_FIRMWARE +extern struct list_head efi_embedded_fw_list; +extern bool efi_embedded_fw_checked; + +static ssize_t trigger_request_platform_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + static const u8 test_data[] = { + 0x55, 0xaa, 0x55, 0xaa, 0x01, 0x02, 0x03, 0x04, + 0x55, 0xaa, 0x55, 0xaa, 0x05, 0x06, 0x07, 0x08, + 0x55, 0xaa, 0x55, 0xaa, 0x10, 0x20, 0x30, 0x40, + 0x55, 0xaa, 0x55, 0xaa, 0x50, 0x60, 0x70, 0x80 + }; + struct efi_embedded_fw efi_embedded_fw; + const struct firmware *firmware = NULL; + bool saved_efi_embedded_fw_checked; + char *name; + int rc; + + name = kstrndup(buf, count, GFP_KERNEL); + if (!name) + return -ENOSPC; + + pr_info("inserting test platform fw '%s'\n", name); + efi_embedded_fw.name = name; + efi_embedded_fw.data = (void *)test_data; + efi_embedded_fw.length = sizeof(test_data); + list_add(&efi_embedded_fw.list, &efi_embedded_fw_list); + saved_efi_embedded_fw_checked = efi_embedded_fw_checked; + efi_embedded_fw_checked = true; + + pr_info("loading '%s'\n", name); + rc = firmware_request_platform(&firmware, name, dev); + if (rc) { + pr_info("load of '%s' failed: %d\n", name, rc); + goto out; + } + if (firmware->size != sizeof(test_data) || + memcmp(firmware->data, test_data, sizeof(test_data)) != 0) { + pr_info("firmware contents mismatch for '%s'\n", name); + rc = -EINVAL; + goto out; + } + pr_info("loaded: %zu\n", firmware->size); + rc = count; + +out: + efi_embedded_fw_checked = saved_efi_embedded_fw_checked; + release_firmware(firmware); + list_del(&efi_embedded_fw.list); + kfree(name); + + return rc; +} +static DEVICE_ATTR_WO(trigger_request_platform); +#endif + static DECLARE_COMPLETION(async_fw_done); static void trigger_async_request_cb(const struct firmware *fw, void *context) @@ -574,7 +729,7 @@ static ssize_t trigger_custom_fallback_store(struct device *dev, mutex_lock(&test_fw_mutex); release_firmware(test_firmware); test_firmware = NULL; - rc = request_firmware_nowait(THIS_MODULE, FW_ACTION_NOHOTPLUG, name, + rc = request_firmware_nowait(THIS_MODULE, FW_ACTION_NOUEVENT, name, dev, GFP_KERNEL, NULL, trigger_async_request_cb); if (rc) { @@ -618,11 +773,21 @@ static int test_fw_run_batch_request(void *data) if (!test_buf) return -ENOSPC; - req->rc = request_firmware_into_buf(&req->fw, - req->name, - req->dev, - test_buf, - TEST_FIRMWARE_BUF_SIZE); + if (test_fw_config->partial) + req->rc = request_partial_firmware_into_buf + (&req->fw, + req->name, + req->dev, + test_buf, + test_fw_config->buf_size, + test_fw_config->file_offset); + else + req->rc = request_firmware_into_buf + (&req->fw, + req->name, + req->dev, + test_buf, + test_fw_config->buf_size); if (!req->fw) kfree(test_buf); } else { @@ -773,8 +938,8 @@ ssize_t trigger_batched_requests_async_store(struct device *dev, pr_info("batched loading '%s' custom fallback mechanism %u times\n", test_fw_config->name, test_fw_config->num_requests); - send_uevent = test_fw_config->send_uevent ? FW_ACTION_HOTPLUG : - FW_ACTION_NOHOTPLUG; + send_uevent = test_fw_config->send_uevent ? FW_ACTION_UEVENT : + FW_ACTION_NOUEVENT; for (i = 0; i < test_fw_config->num_requests; i++) { req = &test_fw_config->reqs[i]; @@ -895,6 +1060,9 @@ static struct attribute *test_dev_attrs[] = { TEST_FW_DEV_ATTR(config_name), TEST_FW_DEV_ATTR(config_num_requests), TEST_FW_DEV_ATTR(config_into_buf), + TEST_FW_DEV_ATTR(config_buf_size), + TEST_FW_DEV_ATTR(config_file_offset), + TEST_FW_DEV_ATTR(config_partial), TEST_FW_DEV_ATTR(config_sync_direct), TEST_FW_DEV_ATTR(config_send_uevent), TEST_FW_DEV_ATTR(config_read_fw_idx), @@ -903,6 +1071,9 @@ static struct attribute *test_dev_attrs[] = { TEST_FW_DEV_ATTR(trigger_request), TEST_FW_DEV_ATTR(trigger_async_request), TEST_FW_DEV_ATTR(trigger_custom_fallback), +#ifdef CONFIG_EFI_EMBEDDED_FIRMWARE + TEST_FW_DEV_ATTR(trigger_request_platform), +#endif /* These use the config and can use the test_result */ TEST_FW_DEV_ATTR(trigger_batched_requests), diff --git a/lib/test_fortify/read_overflow-memchr.c b/lib/test_fortify/read_overflow-memchr.c new file mode 100644 index 000000000000..2743084b32af --- /dev/null +++ b/lib/test_fortify/read_overflow-memchr.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memchr(small, 0x7A, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow-memchr_inv.c b/lib/test_fortify/read_overflow-memchr_inv.c new file mode 100644 index 000000000000..b26e1f1bc217 --- /dev/null +++ b/lib/test_fortify/read_overflow-memchr_inv.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memchr_inv(small, 0x7A, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow-memcmp.c b/lib/test_fortify/read_overflow-memcmp.c new file mode 100644 index 000000000000..d5d301ff64ef --- /dev/null +++ b/lib/test_fortify/read_overflow-memcmp.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memcmp(small, large, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow-memscan.c b/lib/test_fortify/read_overflow-memscan.c new file mode 100644 index 000000000000..c1a97f2df0f0 --- /dev/null +++ b/lib/test_fortify/read_overflow-memscan.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memscan(small, 0x7A, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow2-memcmp.c b/lib/test_fortify/read_overflow2-memcmp.c new file mode 100644 index 000000000000..c6091e640f76 --- /dev/null +++ b/lib/test_fortify/read_overflow2-memcmp.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memcmp(large, small, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow2-memcpy.c b/lib/test_fortify/read_overflow2-memcpy.c new file mode 100644 index 000000000000..07b62e56cf16 --- /dev/null +++ b/lib/test_fortify/read_overflow2-memcpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memcpy(large, instance.buf, sizeof(large)) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow2-memmove.c b/lib/test_fortify/read_overflow2-memmove.c new file mode 100644 index 000000000000..34edfab040a3 --- /dev/null +++ b/lib/test_fortify/read_overflow2-memmove.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memmove(large, instance.buf, sizeof(large)) + +#include "test_fortify.h" diff --git a/lib/test_fortify/test_fortify.h b/lib/test_fortify/test_fortify.h new file mode 100644 index 000000000000..d22664fff197 --- /dev/null +++ b/lib/test_fortify/test_fortify.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include <linux/kernel.h> +#include <linux/printk.h> +#include <linux/slab.h> +#include <linux/string.h> + +void do_fortify_tests(void); + +#define __BUF_SMALL 16 +#define __BUF_LARGE 32 +struct fortify_object { + int a; + char buf[__BUF_SMALL]; + int c; +}; + +#define LITERAL_SMALL "AAAAAAAAAAAAAAA" +#define LITERAL_LARGE "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +const char small_src[__BUF_SMALL] = LITERAL_SMALL; +const char large_src[__BUF_LARGE] = LITERAL_LARGE; + +char small[__BUF_SMALL]; +char large[__BUF_LARGE]; +struct fortify_object instance; +size_t size; + +void do_fortify_tests(void) +{ + /* Normal initializations. */ + memset(&instance, 0x32, sizeof(instance)); + memset(small, 0xA5, sizeof(small)); + memset(large, 0x5A, sizeof(large)); + + TEST; +} diff --git a/lib/test_fortify/write_overflow-memcpy.c b/lib/test_fortify/write_overflow-memcpy.c new file mode 100644 index 000000000000..3b3984e428fb --- /dev/null +++ b/lib/test_fortify/write_overflow-memcpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memcpy(instance.buf, large_src, sizeof(large_src)) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-memmove.c b/lib/test_fortify/write_overflow-memmove.c new file mode 100644 index 000000000000..640437c3b3e0 --- /dev/null +++ b/lib/test_fortify/write_overflow-memmove.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memmove(instance.buf, large_src, sizeof(large_src)) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-memset.c b/lib/test_fortify/write_overflow-memset.c new file mode 100644 index 000000000000..36e34908cfb3 --- /dev/null +++ b/lib/test_fortify/write_overflow-memset.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memset(instance.buf, 0x5A, sizeof(large_src)) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strcpy-lit.c b/lib/test_fortify/write_overflow-strcpy-lit.c new file mode 100644 index 000000000000..51effb3e50f9 --- /dev/null +++ b/lib/test_fortify/write_overflow-strcpy-lit.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strcpy(small, LITERAL_LARGE) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strcpy.c b/lib/test_fortify/write_overflow-strcpy.c new file mode 100644 index 000000000000..84f1c56a64c8 --- /dev/null +++ b/lib/test_fortify/write_overflow-strcpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strcpy(small, large_src) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strlcpy-src.c b/lib/test_fortify/write_overflow-strlcpy-src.c new file mode 100644 index 000000000000..91bf83ebd34a --- /dev/null +++ b/lib/test_fortify/write_overflow-strlcpy-src.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strlcpy(small, large_src, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strlcpy.c b/lib/test_fortify/write_overflow-strlcpy.c new file mode 100644 index 000000000000..1883db7c0cd6 --- /dev/null +++ b/lib/test_fortify/write_overflow-strlcpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strlcpy(instance.buf, large_src, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strncpy-src.c b/lib/test_fortify/write_overflow-strncpy-src.c new file mode 100644 index 000000000000..8dcfb8c788dd --- /dev/null +++ b/lib/test_fortify/write_overflow-strncpy-src.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strncpy(small, large_src, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strncpy.c b/lib/test_fortify/write_overflow-strncpy.c new file mode 100644 index 000000000000..b85f079c815d --- /dev/null +++ b/lib/test_fortify/write_overflow-strncpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strncpy(instance.buf, large_src, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strscpy.c b/lib/test_fortify/write_overflow-strscpy.c new file mode 100644 index 000000000000..38feddf377dc --- /dev/null +++ b/lib/test_fortify/write_overflow-strscpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strscpy(instance.buf, large_src, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fpu.c b/lib/test_fpu.c new file mode 100644 index 000000000000..e82db19fed84 --- /dev/null +++ b/lib/test_fpu.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Test cases for using floating point operations inside a kernel module. + * + * This tests kernel_fpu_begin() and kernel_fpu_end() functions, especially + * when userland has modified the floating point control registers. The kernel + * state might depend on the state set by the userland thread that was active + * before a syscall. + * + * To facilitate the test, this module registers file + * /sys/kernel/debug/selftest_helpers/test_fpu, which when read causes a + * sequence of floating point operations. If the operations fail, either the + * read returns error status or the kernel crashes. + * If the operations succeed, the read returns "1\n". + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/debugfs.h> +#include <asm/fpu/api.h> + +static int test_fpu(void) +{ + /* + * This sequence of operations tests that rounding mode is + * to nearest and that denormal numbers are supported. + * Volatile variables are used to avoid compiler optimizing + * the calculations away. + */ + volatile double a, b, c, d, e, f, g; + + a = 4.0; + b = 1e-15; + c = 1e-310; + + /* Sets precision flag */ + d = a + b; + + /* Result depends on rounding mode */ + e = a + b / 2; + + /* Denormal and very large values */ + f = b / c; + + /* Depends on denormal support */ + g = a + c * f; + + if (d > a && e > a && g > a) + return 0; + else + return -EINVAL; +} + +static int test_fpu_get(void *data, u64 *val) +{ + int status = -EINVAL; + + kernel_fpu_begin(); + status = test_fpu(); + kernel_fpu_end(); + + *val = 1; + return status; +} + +DEFINE_DEBUGFS_ATTRIBUTE(test_fpu_fops, test_fpu_get, NULL, "%lld\n"); +static struct dentry *selftest_dir; + +static int __init test_fpu_init(void) +{ + selftest_dir = debugfs_create_dir("selftest_helpers", NULL); + if (!selftest_dir) + return -ENOMEM; + + debugfs_create_file_unsafe("test_fpu", 0444, selftest_dir, NULL, + &test_fpu_fops); + + return 0; +} + +static void __exit test_fpu_exit(void) +{ + debugfs_remove(selftest_dir); +} + +module_init(test_fpu_init); +module_exit(test_fpu_exit); + +MODULE_LICENSE("GPL"); diff --git a/lib/test_free_pages.c b/lib/test_free_pages.c new file mode 100644 index 000000000000..25ae1ac2624a --- /dev/null +++ b/lib/test_free_pages.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * test_free_pages.c: Check that free_pages() doesn't leak memory + * Copyright (c) 2020 Oracle + * Author: Matthew Wilcox <willy@infradead.org> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/gfp.h> +#include <linux/mm.h> +#include <linux/module.h> + +static void test_free_pages(gfp_t gfp) +{ + unsigned int i; + + for (i = 0; i < 1000 * 1000; i++) { + unsigned long addr = __get_free_pages(gfp, 3); + struct page *page = virt_to_page(addr); + + /* Simulate page cache getting a speculative reference */ + get_page(page); + free_pages(addr, 3); + put_page(page); + } +} + +static int m_in(void) +{ + pr_info("Testing with GFP_KERNEL\n"); + test_free_pages(GFP_KERNEL); + pr_info("Testing with GFP_KERNEL | __GFP_COMP\n"); + test_free_pages(GFP_KERNEL | __GFP_COMP); + pr_info("Test completed\n"); + + return 0; +} + +static void m_ex(void) +{ +} + +module_init(m_in); +module_exit(m_ex); +MODULE_AUTHOR("Matthew Wilcox <willy@infradead.org>"); +MODULE_LICENSE("GPL"); diff --git a/lib/test_hmm.c b/lib/test_hmm.c new file mode 100644 index 000000000000..e2ce8f9b7605 --- /dev/null +++ b/lib/test_hmm.c @@ -0,0 +1,1272 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This is a module to test the HMM (Heterogeneous Memory Management) + * mirror and zone device private memory migration APIs of the kernel. + * Userspace programs can register with the driver to mirror their own address + * space and can use the device to read/write any valid virtual address. + */ +#include <linux/init.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/cdev.h> +#include <linux/device.h> +#include <linux/mutex.h> +#include <linux/rwsem.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/highmem.h> +#include <linux/delay.h> +#include <linux/pagemap.h> +#include <linux/hmm.h> +#include <linux/vmalloc.h> +#include <linux/swap.h> +#include <linux/swapops.h> +#include <linux/sched/mm.h> +#include <linux/platform_device.h> +#include <linux/rmap.h> + +#include "test_hmm_uapi.h" + +#define DMIRROR_NDEVICES 2 +#define DMIRROR_RANGE_FAULT_TIMEOUT 1000 +#define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U) +#define DEVMEM_CHUNKS_RESERVE 16 + +static const struct dev_pagemap_ops dmirror_devmem_ops; +static const struct mmu_interval_notifier_ops dmirror_min_ops; +static dev_t dmirror_dev; + +struct dmirror_device; + +struct dmirror_bounce { + void *ptr; + unsigned long size; + unsigned long addr; + unsigned long cpages; +}; + +#define DPT_XA_TAG_ATOMIC 1UL +#define DPT_XA_TAG_WRITE 3UL + +/* + * Data structure to track address ranges and register for mmu interval + * notifier updates. + */ +struct dmirror_interval { + struct mmu_interval_notifier notifier; + struct dmirror *dmirror; +}; + +/* + * Data attached to the open device file. + * Note that it might be shared after a fork(). + */ +struct dmirror { + struct dmirror_device *mdevice; + struct xarray pt; + struct mmu_interval_notifier notifier; + struct mutex mutex; +}; + +/* + * ZONE_DEVICE pages for migration and simulating device memory. + */ +struct dmirror_chunk { + struct dev_pagemap pagemap; + struct dmirror_device *mdevice; +}; + +/* + * Per device data. + */ +struct dmirror_device { + struct cdev cdevice; + struct hmm_devmem *devmem; + + unsigned int devmem_capacity; + unsigned int devmem_count; + struct dmirror_chunk **devmem_chunks; + struct mutex devmem_lock; /* protects the above */ + + unsigned long calloc; + unsigned long cfree; + struct page *free_pages; + spinlock_t lock; /* protects the above */ +}; + +static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; + +static int dmirror_bounce_init(struct dmirror_bounce *bounce, + unsigned long addr, + unsigned long size) +{ + bounce->addr = addr; + bounce->size = size; + bounce->cpages = 0; + bounce->ptr = vmalloc(size); + if (!bounce->ptr) + return -ENOMEM; + return 0; +} + +static void dmirror_bounce_fini(struct dmirror_bounce *bounce) +{ + vfree(bounce->ptr); +} + +static int dmirror_fops_open(struct inode *inode, struct file *filp) +{ + struct cdev *cdev = inode->i_cdev; + struct dmirror *dmirror; + int ret; + + /* Mirror this process address space */ + dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL); + if (dmirror == NULL) + return -ENOMEM; + + dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); + mutex_init(&dmirror->mutex); + xa_init(&dmirror->pt); + + ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, + 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); + if (ret) { + kfree(dmirror); + return ret; + } + + filp->private_data = dmirror; + return 0; +} + +static int dmirror_fops_release(struct inode *inode, struct file *filp) +{ + struct dmirror *dmirror = filp->private_data; + + mmu_interval_notifier_remove(&dmirror->notifier); + xa_destroy(&dmirror->pt); + kfree(dmirror); + return 0; +} + +static struct dmirror_device *dmirror_page_to_device(struct page *page) + +{ + return container_of(page->pgmap, struct dmirror_chunk, + pagemap)->mdevice; +} + +static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) +{ + unsigned long *pfns = range->hmm_pfns; + unsigned long pfn; + + for (pfn = (range->start >> PAGE_SHIFT); + pfn < (range->end >> PAGE_SHIFT); + pfn++, pfns++) { + struct page *page; + void *entry; + + /* + * Since we asked for hmm_range_fault() to populate pages, + * it shouldn't return an error entry on success. + */ + WARN_ON(*pfns & HMM_PFN_ERROR); + WARN_ON(!(*pfns & HMM_PFN_VALID)); + + page = hmm_pfn_to_page(*pfns); + WARN_ON(!page); + + entry = page; + if (*pfns & HMM_PFN_WRITE) + entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); + else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) + return -EFAULT; + entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); + if (xa_is_err(entry)) + return xa_err(entry); + } + + return 0; +} + +static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, + unsigned long end) +{ + unsigned long pfn; + void *entry; + + /* + * The XArray doesn't hold references to pages since it relies on + * the mmu notifier to clear page pointers when they become stale. + * Therefore, it is OK to just clear the entry. + */ + xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, + end >> PAGE_SHIFT) + xa_erase(&dmirror->pt, pfn); +} + +static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, + const struct mmu_notifier_range *range, + unsigned long cur_seq) +{ + struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); + + /* + * Ignore invalidation callbacks for device private pages since + * the invalidation is handled as part of the migration process. + */ + if (range->event == MMU_NOTIFY_MIGRATE && + range->owner == dmirror->mdevice) + return true; + + if (mmu_notifier_range_blockable(range)) + mutex_lock(&dmirror->mutex); + else if (!mutex_trylock(&dmirror->mutex)) + return false; + + mmu_interval_set_seq(mni, cur_seq); + dmirror_do_update(dmirror, range->start, range->end); + + mutex_unlock(&dmirror->mutex); + return true; +} + +static const struct mmu_interval_notifier_ops dmirror_min_ops = { + .invalidate = dmirror_interval_invalidate, +}; + +static int dmirror_range_fault(struct dmirror *dmirror, + struct hmm_range *range) +{ + struct mm_struct *mm = dmirror->notifier.mm; + unsigned long timeout = + jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + int ret; + + while (true) { + if (time_after(jiffies, timeout)) { + ret = -EBUSY; + goto out; + } + + range->notifier_seq = mmu_interval_read_begin(range->notifier); + mmap_read_lock(mm); + ret = hmm_range_fault(range); + mmap_read_unlock(mm); + if (ret) { + if (ret == -EBUSY) + continue; + goto out; + } + + mutex_lock(&dmirror->mutex); + if (mmu_interval_read_retry(range->notifier, + range->notifier_seq)) { + mutex_unlock(&dmirror->mutex); + continue; + } + break; + } + + ret = dmirror_do_fault(dmirror, range); + + mutex_unlock(&dmirror->mutex); +out: + return ret; +} + +static int dmirror_fault(struct dmirror *dmirror, unsigned long start, + unsigned long end, bool write) +{ + struct mm_struct *mm = dmirror->notifier.mm; + unsigned long addr; + unsigned long pfns[64]; + struct hmm_range range = { + .notifier = &dmirror->notifier, + .hmm_pfns = pfns, + .pfn_flags_mask = 0, + .default_flags = + HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), + .dev_private_owner = dmirror->mdevice, + }; + int ret = 0; + + /* Since the mm is for the mirrored process, get a reference first. */ + if (!mmget_not_zero(mm)) + return 0; + + for (addr = start; addr < end; addr = range.end) { + range.start = addr; + range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); + + ret = dmirror_range_fault(dmirror, &range); + if (ret) + break; + } + + mmput(mm); + return ret; +} + +static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, + unsigned long end, struct dmirror_bounce *bounce) +{ + unsigned long pfn; + void *ptr; + + ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); + + for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { + void *entry; + struct page *page; + void *tmp; + + entry = xa_load(&dmirror->pt, pfn); + page = xa_untag_pointer(entry); + if (!page) + return -ENOENT; + + tmp = kmap(page); + memcpy(ptr, tmp, PAGE_SIZE); + kunmap(page); + + ptr += PAGE_SIZE; + bounce->cpages++; + } + + return 0; +} + +static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) +{ + struct dmirror_bounce bounce; + unsigned long start, end; + unsigned long size = cmd->npages << PAGE_SHIFT; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + + while (1) { + mutex_lock(&dmirror->mutex); + ret = dmirror_do_read(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret != -ENOENT) + break; + + start = cmd->addr + (bounce.cpages << PAGE_SHIFT); + ret = dmirror_fault(dmirror, start, end, false); + if (ret) + break; + cmd->faults++; + } + + if (ret == 0) { + if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, + bounce.size)) + ret = -EFAULT; + } + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; +} + +static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, + unsigned long end, struct dmirror_bounce *bounce) +{ + unsigned long pfn; + void *ptr; + + ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); + + for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { + void *entry; + struct page *page; + void *tmp; + + entry = xa_load(&dmirror->pt, pfn); + page = xa_untag_pointer(entry); + if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) + return -ENOENT; + + tmp = kmap(page); + memcpy(tmp, ptr, PAGE_SIZE); + kunmap(page); + + ptr += PAGE_SIZE; + bounce->cpages++; + } + + return 0; +} + +static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) +{ + struct dmirror_bounce bounce; + unsigned long start, end; + unsigned long size = cmd->npages << PAGE_SHIFT; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), + bounce.size)) { + ret = -EFAULT; + goto fini; + } + + while (1) { + mutex_lock(&dmirror->mutex); + ret = dmirror_do_write(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret != -ENOENT) + break; + + start = cmd->addr + (bounce.cpages << PAGE_SHIFT); + ret = dmirror_fault(dmirror, start, end, true); + if (ret) + break; + cmd->faults++; + } + +fini: + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; +} + +static bool dmirror_allocate_chunk(struct dmirror_device *mdevice, + struct page **ppage) +{ + struct dmirror_chunk *devmem; + struct resource *res; + unsigned long pfn; + unsigned long pfn_first; + unsigned long pfn_last; + void *ptr; + + devmem = kzalloc(sizeof(*devmem), GFP_KERNEL); + if (!devmem) + return false; + + res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, + "hmm_dmirror"); + if (IS_ERR(res)) + goto err_devmem; + + devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; + devmem->pagemap.range.start = res->start; + devmem->pagemap.range.end = res->end; + devmem->pagemap.nr_range = 1; + devmem->pagemap.ops = &dmirror_devmem_ops; + devmem->pagemap.owner = mdevice; + + mutex_lock(&mdevice->devmem_lock); + + if (mdevice->devmem_count == mdevice->devmem_capacity) { + struct dmirror_chunk **new_chunks; + unsigned int new_capacity; + + new_capacity = mdevice->devmem_capacity + + DEVMEM_CHUNKS_RESERVE; + new_chunks = krealloc(mdevice->devmem_chunks, + sizeof(new_chunks[0]) * new_capacity, + GFP_KERNEL); + if (!new_chunks) + goto err_release; + mdevice->devmem_capacity = new_capacity; + mdevice->devmem_chunks = new_chunks; + } + + ptr = memremap_pages(&devmem->pagemap, numa_node_id()); + if (IS_ERR(ptr)) + goto err_release; + + devmem->mdevice = mdevice; + pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT; + pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT); + mdevice->devmem_chunks[mdevice->devmem_count++] = devmem; + + mutex_unlock(&mdevice->devmem_lock); + + pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", + DEVMEM_CHUNK_SIZE / (1024 * 1024), + mdevice->devmem_count, + mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), + pfn_first, pfn_last); + + spin_lock(&mdevice->lock); + for (pfn = pfn_first; pfn < pfn_last; pfn++) { + struct page *page = pfn_to_page(pfn); + + page->zone_device_data = mdevice->free_pages; + mdevice->free_pages = page; + } + if (ppage) { + *ppage = mdevice->free_pages; + mdevice->free_pages = (*ppage)->zone_device_data; + mdevice->calloc++; + } + spin_unlock(&mdevice->lock); + + return true; + +err_release: + mutex_unlock(&mdevice->devmem_lock); + release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range)); +err_devmem: + kfree(devmem); + + return false; +} + +static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice) +{ + struct page *dpage = NULL; + struct page *rpage; + + /* + * This is a fake device so we alloc real system memory to store + * our device memory. + */ + rpage = alloc_page(GFP_HIGHUSER); + if (!rpage) + return NULL; + + spin_lock(&mdevice->lock); + + if (mdevice->free_pages) { + dpage = mdevice->free_pages; + mdevice->free_pages = dpage->zone_device_data; + mdevice->calloc++; + spin_unlock(&mdevice->lock); + } else { + spin_unlock(&mdevice->lock); + if (!dmirror_allocate_chunk(mdevice, &dpage)) + goto error; + } + + dpage->zone_device_data = rpage; + get_page(dpage); + lock_page(dpage); + return dpage; + +error: + __free_page(rpage); + return NULL; +} + +static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, + struct dmirror *dmirror) +{ + struct dmirror_device *mdevice = dmirror->mdevice; + const unsigned long *src = args->src; + unsigned long *dst = args->dst; + unsigned long addr; + + for (addr = args->start; addr < args->end; addr += PAGE_SIZE, + src++, dst++) { + struct page *spage; + struct page *dpage; + struct page *rpage; + + if (!(*src & MIGRATE_PFN_MIGRATE)) + continue; + + /* + * Note that spage might be NULL which is OK since it is an + * unallocated pte_none() or read-only zero page. + */ + spage = migrate_pfn_to_page(*src); + + dpage = dmirror_devmem_alloc_page(mdevice); + if (!dpage) + continue; + + rpage = dpage->zone_device_data; + if (spage) + copy_highpage(rpage, spage); + else + clear_highpage(rpage); + + /* + * Normally, a device would use the page->zone_device_data to + * point to the mirror but here we use it to hold the page for + * the simulated device memory and that page holds the pointer + * to the mirror. + */ + rpage->zone_device_data = dmirror; + + *dst = migrate_pfn(page_to_pfn(dpage)); + if ((*src & MIGRATE_PFN_WRITE) || + (!spage && args->vma->vm_flags & VM_WRITE)) + *dst |= MIGRATE_PFN_WRITE; + } +} + +static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, + unsigned long end) +{ + unsigned long pfn; + + for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { + void *entry; + + entry = xa_load(&dmirror->pt, pfn); + if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) + return -EPERM; + } + + return 0; +} + +static int dmirror_atomic_map(unsigned long start, unsigned long end, + struct page **pages, struct dmirror *dmirror) +{ + unsigned long pfn, mapped = 0; + int i; + + /* Map the migrated pages into the device's page tables. */ + mutex_lock(&dmirror->mutex); + + for (i = 0, pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, i++) { + void *entry; + + if (!pages[i]) + continue; + + entry = pages[i]; + entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC); + entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); + if (xa_is_err(entry)) { + mutex_unlock(&dmirror->mutex); + return xa_err(entry); + } + + mapped++; + } + + mutex_unlock(&dmirror->mutex); + return mapped; +} + +static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, + struct dmirror *dmirror) +{ + unsigned long start = args->start; + unsigned long end = args->end; + const unsigned long *src = args->src; + const unsigned long *dst = args->dst; + unsigned long pfn; + + /* Map the migrated pages into the device's page tables. */ + mutex_lock(&dmirror->mutex); + + for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, + src++, dst++) { + struct page *dpage; + void *entry; + + if (!(*src & MIGRATE_PFN_MIGRATE)) + continue; + + dpage = migrate_pfn_to_page(*dst); + if (!dpage) + continue; + + /* + * Store the page that holds the data so the page table + * doesn't have to deal with ZONE_DEVICE private pages. + */ + entry = dpage->zone_device_data; + if (*dst & MIGRATE_PFN_WRITE) + entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); + entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); + if (xa_is_err(entry)) { + mutex_unlock(&dmirror->mutex); + return xa_err(entry); + } + } + + mutex_unlock(&dmirror->mutex); + return 0; +} + +static int dmirror_exclusive(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd) +{ + unsigned long start, end, addr; + unsigned long size = cmd->npages << PAGE_SHIFT; + struct mm_struct *mm = dmirror->notifier.mm; + struct page *pages[64]; + struct dmirror_bounce bounce; + unsigned long next; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + /* Since the mm is for the mirrored process, get a reference first. */ + if (!mmget_not_zero(mm)) + return -EINVAL; + + mmap_read_lock(mm); + for (addr = start; addr < end; addr = next) { + unsigned long mapped; + int i; + + if (end < addr + (ARRAY_SIZE(pages) << PAGE_SHIFT)) + next = end; + else + next = addr + (ARRAY_SIZE(pages) << PAGE_SHIFT); + + ret = make_device_exclusive_range(mm, addr, next, pages, NULL); + mapped = dmirror_atomic_map(addr, next, pages, dmirror); + for (i = 0; i < ret; i++) { + if (pages[i]) { + unlock_page(pages[i]); + put_page(pages[i]); + } + } + + if (addr + (mapped << PAGE_SHIFT) < next) { + mmap_read_unlock(mm); + mmput(mm); + return -EBUSY; + } + } + mmap_read_unlock(mm); + mmput(mm); + + /* Return the migrated data for verification. */ + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + mutex_lock(&dmirror->mutex); + ret = dmirror_do_read(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret == 0) { + if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, + bounce.size)) + ret = -EFAULT; + } + + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; +} + +static int dmirror_migrate(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd) +{ + unsigned long start, end, addr; + unsigned long size = cmd->npages << PAGE_SHIFT; + struct mm_struct *mm = dmirror->notifier.mm; + struct vm_area_struct *vma; + unsigned long src_pfns[64]; + unsigned long dst_pfns[64]; + struct dmirror_bounce bounce; + struct migrate_vma args; + unsigned long next; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + /* Since the mm is for the mirrored process, get a reference first. */ + if (!mmget_not_zero(mm)) + return -EINVAL; + + mmap_read_lock(mm); + for (addr = start; addr < end; addr = next) { + vma = vma_lookup(mm, addr); + if (!vma || !(vma->vm_flags & VM_READ)) { + ret = -EINVAL; + goto out; + } + next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT)); + if (next > vma->vm_end) + next = vma->vm_end; + + args.vma = vma; + args.src = src_pfns; + args.dst = dst_pfns; + args.start = addr; + args.end = next; + args.pgmap_owner = dmirror->mdevice; + args.flags = MIGRATE_VMA_SELECT_SYSTEM; + ret = migrate_vma_setup(&args); + if (ret) + goto out; + + dmirror_migrate_alloc_and_copy(&args, dmirror); + migrate_vma_pages(&args); + dmirror_migrate_finalize_and_map(&args, dmirror); + migrate_vma_finalize(&args); + } + mmap_read_unlock(mm); + mmput(mm); + + /* Return the migrated data for verification. */ + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + mutex_lock(&dmirror->mutex); + ret = dmirror_do_read(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret == 0) { + if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, + bounce.size)) + ret = -EFAULT; + } + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; + +out: + mmap_read_unlock(mm); + mmput(mm); + return ret; +} + +static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, + unsigned char *perm, unsigned long entry) +{ + struct page *page; + + if (entry & HMM_PFN_ERROR) { + *perm = HMM_DMIRROR_PROT_ERROR; + return; + } + if (!(entry & HMM_PFN_VALID)) { + *perm = HMM_DMIRROR_PROT_NONE; + return; + } + + page = hmm_pfn_to_page(entry); + if (is_device_private_page(page)) { + /* Is the page migrated to this device or some other? */ + if (dmirror->mdevice == dmirror_page_to_device(page)) + *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; + else + *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; + } else if (is_zero_pfn(page_to_pfn(page))) + *perm = HMM_DMIRROR_PROT_ZERO; + else + *perm = HMM_DMIRROR_PROT_NONE; + if (entry & HMM_PFN_WRITE) + *perm |= HMM_DMIRROR_PROT_WRITE; + else + *perm |= HMM_DMIRROR_PROT_READ; + if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT) + *perm |= HMM_DMIRROR_PROT_PMD; + else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT) + *perm |= HMM_DMIRROR_PROT_PUD; +} + +static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, + const struct mmu_notifier_range *range, + unsigned long cur_seq) +{ + struct dmirror_interval *dmi = + container_of(mni, struct dmirror_interval, notifier); + struct dmirror *dmirror = dmi->dmirror; + + if (mmu_notifier_range_blockable(range)) + mutex_lock(&dmirror->mutex); + else if (!mutex_trylock(&dmirror->mutex)) + return false; + + /* + * Snapshots only need to set the sequence number since any + * invalidation in the interval invalidates the whole snapshot. + */ + mmu_interval_set_seq(mni, cur_seq); + + mutex_unlock(&dmirror->mutex); + return true; +} + +static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { + .invalidate = dmirror_snapshot_invalidate, +}; + +static int dmirror_range_snapshot(struct dmirror *dmirror, + struct hmm_range *range, + unsigned char *perm) +{ + struct mm_struct *mm = dmirror->notifier.mm; + struct dmirror_interval notifier; + unsigned long timeout = + jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + unsigned long i; + unsigned long n; + int ret = 0; + + notifier.dmirror = dmirror; + range->notifier = ¬ifier.notifier; + + ret = mmu_interval_notifier_insert(range->notifier, mm, + range->start, range->end - range->start, + &dmirror_mrn_ops); + if (ret) + return ret; + + while (true) { + if (time_after(jiffies, timeout)) { + ret = -EBUSY; + goto out; + } + + range->notifier_seq = mmu_interval_read_begin(range->notifier); + + mmap_read_lock(mm); + ret = hmm_range_fault(range); + mmap_read_unlock(mm); + if (ret) { + if (ret == -EBUSY) + continue; + goto out; + } + + mutex_lock(&dmirror->mutex); + if (mmu_interval_read_retry(range->notifier, + range->notifier_seq)) { + mutex_unlock(&dmirror->mutex); + continue; + } + break; + } + + n = (range->end - range->start) >> PAGE_SHIFT; + for (i = 0; i < n; i++) + dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); + + mutex_unlock(&dmirror->mutex); +out: + mmu_interval_notifier_remove(range->notifier); + return ret; +} + +static int dmirror_snapshot(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd) +{ + struct mm_struct *mm = dmirror->notifier.mm; + unsigned long start, end; + unsigned long size = cmd->npages << PAGE_SHIFT; + unsigned long addr; + unsigned long next; + unsigned long pfns[64]; + unsigned char perm[64]; + char __user *uptr; + struct hmm_range range = { + .hmm_pfns = pfns, + .dev_private_owner = dmirror->mdevice, + }; + int ret = 0; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + /* Since the mm is for the mirrored process, get a reference first. */ + if (!mmget_not_zero(mm)) + return -EINVAL; + + /* + * Register a temporary notifier to detect invalidations even if it + * overlaps with other mmu_interval_notifiers. + */ + uptr = u64_to_user_ptr(cmd->ptr); + for (addr = start; addr < end; addr = next) { + unsigned long n; + + next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); + range.start = addr; + range.end = next; + + ret = dmirror_range_snapshot(dmirror, &range, perm); + if (ret) + break; + + n = (range.end - range.start) >> PAGE_SHIFT; + if (copy_to_user(uptr, perm, n)) { + ret = -EFAULT; + break; + } + + cmd->cpages += n; + uptr += n; + } + mmput(mm); + + return ret; +} + +static long dmirror_fops_unlocked_ioctl(struct file *filp, + unsigned int command, + unsigned long arg) +{ + void __user *uarg = (void __user *)arg; + struct hmm_dmirror_cmd cmd; + struct dmirror *dmirror; + int ret; + + dmirror = filp->private_data; + if (!dmirror) + return -EINVAL; + + if (copy_from_user(&cmd, uarg, sizeof(cmd))) + return -EFAULT; + + if (cmd.addr & ~PAGE_MASK) + return -EINVAL; + if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) + return -EINVAL; + + cmd.cpages = 0; + cmd.faults = 0; + + switch (command) { + case HMM_DMIRROR_READ: + ret = dmirror_read(dmirror, &cmd); + break; + + case HMM_DMIRROR_WRITE: + ret = dmirror_write(dmirror, &cmd); + break; + + case HMM_DMIRROR_MIGRATE: + ret = dmirror_migrate(dmirror, &cmd); + break; + + case HMM_DMIRROR_EXCLUSIVE: + ret = dmirror_exclusive(dmirror, &cmd); + break; + + case HMM_DMIRROR_CHECK_EXCLUSIVE: + ret = dmirror_check_atomic(dmirror, cmd.addr, + cmd.addr + (cmd.npages << PAGE_SHIFT)); + break; + + case HMM_DMIRROR_SNAPSHOT: + ret = dmirror_snapshot(dmirror, &cmd); + break; + + default: + return -EINVAL; + } + if (ret) + return ret; + + if (copy_to_user(uarg, &cmd, sizeof(cmd))) + return -EFAULT; + + return 0; +} + +static const struct file_operations dmirror_fops = { + .open = dmirror_fops_open, + .release = dmirror_fops_release, + .unlocked_ioctl = dmirror_fops_unlocked_ioctl, + .llseek = default_llseek, + .owner = THIS_MODULE, +}; + +static void dmirror_devmem_free(struct page *page) +{ + struct page *rpage = page->zone_device_data; + struct dmirror_device *mdevice; + + if (rpage) + __free_page(rpage); + + mdevice = dmirror_page_to_device(page); + + spin_lock(&mdevice->lock); + mdevice->cfree++; + page->zone_device_data = mdevice->free_pages; + mdevice->free_pages = page; + spin_unlock(&mdevice->lock); +} + +static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, + struct dmirror *dmirror) +{ + const unsigned long *src = args->src; + unsigned long *dst = args->dst; + unsigned long start = args->start; + unsigned long end = args->end; + unsigned long addr; + + for (addr = start; addr < end; addr += PAGE_SIZE, + src++, dst++) { + struct page *dpage, *spage; + + spage = migrate_pfn_to_page(*src); + if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) + continue; + spage = spage->zone_device_data; + + dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); + if (!dpage) + continue; + + lock_page(dpage); + xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); + copy_highpage(dpage, spage); + *dst = migrate_pfn(page_to_pfn(dpage)); + if (*src & MIGRATE_PFN_WRITE) + *dst |= MIGRATE_PFN_WRITE; + } + return 0; +} + +static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) +{ + struct migrate_vma args; + unsigned long src_pfns; + unsigned long dst_pfns; + struct page *rpage; + struct dmirror *dmirror; + vm_fault_t ret; + + /* + * Normally, a device would use the page->zone_device_data to point to + * the mirror but here we use it to hold the page for the simulated + * device memory and that page holds the pointer to the mirror. + */ + rpage = vmf->page->zone_device_data; + dmirror = rpage->zone_device_data; + + /* FIXME demonstrate how we can adjust migrate range */ + args.vma = vmf->vma; + args.start = vmf->address; + args.end = args.start + PAGE_SIZE; + args.src = &src_pfns; + args.dst = &dst_pfns; + args.pgmap_owner = dmirror->mdevice; + args.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE; + + if (migrate_vma_setup(&args)) + return VM_FAULT_SIGBUS; + + ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); + if (ret) + return ret; + migrate_vma_pages(&args); + /* + * No device finalize step is needed since + * dmirror_devmem_fault_alloc_and_copy() will have already + * invalidated the device page table. + */ + migrate_vma_finalize(&args); + return 0; +} + +static const struct dev_pagemap_ops dmirror_devmem_ops = { + .page_free = dmirror_devmem_free, + .migrate_to_ram = dmirror_devmem_fault, +}; + +static int dmirror_device_init(struct dmirror_device *mdevice, int id) +{ + dev_t dev; + int ret; + + dev = MKDEV(MAJOR(dmirror_dev), id); + mutex_init(&mdevice->devmem_lock); + spin_lock_init(&mdevice->lock); + + cdev_init(&mdevice->cdevice, &dmirror_fops); + mdevice->cdevice.owner = THIS_MODULE; + ret = cdev_add(&mdevice->cdevice, dev, 1); + if (ret) + return ret; + + /* Build a list of free ZONE_DEVICE private struct pages */ + dmirror_allocate_chunk(mdevice, NULL); + + return 0; +} + +static void dmirror_device_remove(struct dmirror_device *mdevice) +{ + unsigned int i; + + if (mdevice->devmem_chunks) { + for (i = 0; i < mdevice->devmem_count; i++) { + struct dmirror_chunk *devmem = + mdevice->devmem_chunks[i]; + + memunmap_pages(&devmem->pagemap); + release_mem_region(devmem->pagemap.range.start, + range_len(&devmem->pagemap.range)); + kfree(devmem); + } + kfree(mdevice->devmem_chunks); + } + + cdev_del(&mdevice->cdevice); +} + +static int __init hmm_dmirror_init(void) +{ + int ret; + int id; + + ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, + "HMM_DMIRROR"); + if (ret) + goto err_unreg; + + for (id = 0; id < DMIRROR_NDEVICES; id++) { + ret = dmirror_device_init(dmirror_devices + id, id); + if (ret) + goto err_chrdev; + } + + pr_info("HMM test module loaded. This is only for testing HMM.\n"); + return 0; + +err_chrdev: + while (--id >= 0) + dmirror_device_remove(dmirror_devices + id); + unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); +err_unreg: + return ret; +} + +static void __exit hmm_dmirror_exit(void) +{ + int id; + + for (id = 0; id < DMIRROR_NDEVICES; id++) + dmirror_device_remove(dmirror_devices + id); + unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); +} + +module_init(hmm_dmirror_init); +module_exit(hmm_dmirror_exit); +MODULE_LICENSE("GPL"); diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h new file mode 100644 index 000000000000..f14dea5dcd06 --- /dev/null +++ b/lib/test_hmm_uapi.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * This is a module to test the HMM (Heterogeneous Memory Management) API + * of the kernel. It allows a userspace program to expose its entire address + * space through the HMM test module device file. + */ +#ifndef _LIB_TEST_HMM_UAPI_H +#define _LIB_TEST_HMM_UAPI_H + +#include <linux/types.h> +#include <linux/ioctl.h> + +/* + * Structure to pass to the HMM test driver to mimic a device accessing + * system memory and ZONE_DEVICE private memory through device page tables. + * + * @addr: (in) user address the device will read/write + * @ptr: (in) user address where device data is copied to/from + * @npages: (in) number of pages to read/write + * @cpages: (out) number of pages copied + * @faults: (out) number of device page faults seen + */ +struct hmm_dmirror_cmd { + __u64 addr; + __u64 ptr; + __u64 npages; + __u64 cpages; + __u64 faults; +}; + +/* Expose the address space of the calling process through hmm device file */ +#define HMM_DMIRROR_READ _IOWR('H', 0x00, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_WRITE _IOWR('H', 0x01, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_MIGRATE _IOWR('H', 0x02, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_SNAPSHOT _IOWR('H', 0x03, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_EXCLUSIVE _IOWR('H', 0x04, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x05, struct hmm_dmirror_cmd) + +/* + * Values returned in hmm_dmirror_cmd.ptr for HMM_DMIRROR_SNAPSHOT. + * HMM_DMIRROR_PROT_ERROR: no valid mirror PTE for this page + * HMM_DMIRROR_PROT_NONE: unpopulated PTE or PTE with no access + * HMM_DMIRROR_PROT_READ: read-only PTE + * HMM_DMIRROR_PROT_WRITE: read/write PTE + * HMM_DMIRROR_PROT_PMD: PMD sized page is fully mapped by same permissions + * HMM_DMIRROR_PROT_PUD: PUD sized page is fully mapped by same permissions + * HMM_DMIRROR_PROT_ZERO: special read-only zero page + * HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL: Migrated device private page on the + * device the ioctl() is made + * HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE: Migrated device private page on some + * other device + */ +enum { + HMM_DMIRROR_PROT_ERROR = 0xFF, + HMM_DMIRROR_PROT_NONE = 0x00, + HMM_DMIRROR_PROT_READ = 0x01, + HMM_DMIRROR_PROT_WRITE = 0x02, + HMM_DMIRROR_PROT_PMD = 0x04, + HMM_DMIRROR_PROT_PUD = 0x08, + HMM_DMIRROR_PROT_ZERO = 0x10, + HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL = 0x20, + HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE = 0x30, +}; + +#endif /* _LIB_TEST_HMM_UAPI_H */ diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 328d33beae36..67ed689a0b1b 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -5,8 +5,6 @@ * Author: Andrey Ryabinin <a.ryabinin@samsung.com> */ -#define pr_fmt(fmt) "kasan test: %s " fmt, __func__ - #include <linux/bitops.h> #include <linux/delay.h> #include <linux/kasan.h> @@ -15,6 +13,7 @@ #include <linux/mman.h> #include <linux/module.h> #include <linux/printk.h> +#include <linux/random.h> #include <linux/slab.h> #include <linux/string.h> #include <linux/uaccess.h> @@ -23,385 +22,631 @@ #include <asm/page.h> +#include <kunit/test.h> + +#include "../mm/kasan/kasan.h" + +#define OOB_TAG_OFF (IS_ENABLED(CONFIG_KASAN_GENERIC) ? 0 : KASAN_GRANULE_SIZE) + /* - * Note: test functions are marked noinline so that their names appear in - * reports. + * Some tests use these global variables to store return values from function + * calls that could otherwise be eliminated by the compiler as dead code. */ +void *kasan_ptr_result; +int kasan_int_result; + +static struct kunit_resource resource; +static struct kunit_kasan_expectation fail_data; +static bool multishot; -static noinline void __init kmalloc_oob_right(void) +/* + * Temporarily enable multi-shot mode. Otherwise, KASAN would only report the + * first detected bug and panic the kernel if panic_on_warn is enabled. For + * hardware tag-based KASAN also allow tag checking to be reenabled for each + * test, see the comment for KUNIT_EXPECT_KASAN_FAIL(). + */ +static int kasan_test_init(struct kunit *test) +{ + if (!kasan_enabled()) { + kunit_err(test, "can't run KASAN tests with KASAN disabled"); + return -1; + } + + multishot = kasan_save_enable_multi_shot(); + fail_data.report_found = false; + kunit_add_named_resource(test, NULL, NULL, &resource, + "kasan_data", &fail_data); + return 0; +} + +static void kasan_test_exit(struct kunit *test) +{ + kasan_restore_multi_shot(multishot); + KUNIT_EXPECT_FALSE(test, fail_data.report_found); +} + +/** + * KUNIT_EXPECT_KASAN_FAIL() - check that the executed expression produces a + * KASAN report; causes a test failure otherwise. This relies on a KUnit + * resource named "kasan_data". Do not use this name for KUnit resources + * outside of KASAN tests. + * + * For hardware tag-based KASAN in sync mode, when a tag fault happens, tag + * checking is auto-disabled. When this happens, this test handler reenables + * tag checking. As tag checking can be only disabled or enabled per CPU, + * this handler disables migration (preemption). + * + * Since the compiler doesn't see that the expression can change the fail_data + * fields, it can reorder or optimize away the accesses to those fields. + * Use READ/WRITE_ONCE() for the accesses and compiler barriers around the + * expression to prevent that. + * + * In between KUNIT_EXPECT_KASAN_FAIL checks, fail_data.report_found is kept as + * false. This allows detecting KASAN reports that happen outside of the checks + * by asserting !fail_data.report_found at the start of KUNIT_EXPECT_KASAN_FAIL + * and in kasan_test_exit. + */ +#define KUNIT_EXPECT_KASAN_FAIL(test, expression) do { \ + if (IS_ENABLED(CONFIG_KASAN_HW_TAGS) && \ + kasan_sync_fault_possible()) \ + migrate_disable(); \ + KUNIT_EXPECT_FALSE(test, READ_ONCE(fail_data.report_found)); \ + barrier(); \ + expression; \ + barrier(); \ + if (!READ_ONCE(fail_data.report_found)) { \ + KUNIT_FAIL(test, KUNIT_SUBTEST_INDENT "KASAN failure " \ + "expected in \"" #expression \ + "\", but none occurred"); \ + } \ + if (IS_ENABLED(CONFIG_KASAN_HW_TAGS)) { \ + if (READ_ONCE(fail_data.report_found)) \ + kasan_enable_tagging_sync(); \ + migrate_enable(); \ + } \ + WRITE_ONCE(fail_data.report_found, false); \ +} while (0) + +#define KASAN_TEST_NEEDS_CONFIG_ON(test, config) do { \ + if (!IS_ENABLED(config)) \ + kunit_skip((test), "Test requires " #config "=y"); \ +} while (0) + +#define KASAN_TEST_NEEDS_CONFIG_OFF(test, config) do { \ + if (IS_ENABLED(config)) \ + kunit_skip((test), "Test requires " #config "=n"); \ +} while (0) + +static void kmalloc_oob_right(struct kunit *test) { char *ptr; - size_t size = 123; + size_t size = 128 - KASAN_GRANULE_SIZE - 5; - pr_info("out-of-bounds to right\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + + /* + * An unaligned access past the requested kmalloc size. + * Only generic KASAN can precisely detect these. + */ + if (IS_ENABLED(CONFIG_KASAN_GENERIC)) + KUNIT_EXPECT_KASAN_FAIL(test, ptr[size] = 'x'); + + /* + * An aligned access into the first out-of-bounds granule that falls + * within the aligned kmalloc object. + */ + KUNIT_EXPECT_KASAN_FAIL(test, ptr[size + 5] = 'y'); + + /* Out-of-bounds access past the aligned kmalloc object. */ + KUNIT_EXPECT_KASAN_FAIL(test, ptr[0] = + ptr[size + KASAN_GRANULE_SIZE + 5]); - ptr[size] = 'x'; kfree(ptr); } -static noinline void __init kmalloc_oob_left(void) +static void kmalloc_oob_left(struct kunit *test) { char *ptr; size_t size = 15; - pr_info("out-of-bounds to left\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - *ptr = *(ptr - 1); + KUNIT_EXPECT_KASAN_FAIL(test, *ptr = *(ptr - 1)); kfree(ptr); } -static noinline void __init kmalloc_node_oob_right(void) +static void kmalloc_node_oob_right(struct kunit *test) { char *ptr; size_t size = 4096; - pr_info("kmalloc_node(): out-of-bounds to right\n"); ptr = kmalloc_node(size, GFP_KERNEL, 0); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - ptr[size] = 0; + KUNIT_EXPECT_KASAN_FAIL(test, ptr[0] = ptr[size]); kfree(ptr); } -#ifdef CONFIG_SLUB -static noinline void __init kmalloc_pagealloc_oob_right(void) +/* + * These kmalloc_pagealloc_* tests try allocating a memory chunk that doesn't + * fit into a slab cache and therefore is allocated via the page allocator + * fallback. Since this kind of fallback is only implemented for SLUB, these + * tests are limited to that allocator. + */ +static void kmalloc_pagealloc_oob_right(struct kunit *test) { char *ptr; size_t size = KMALLOC_MAX_CACHE_SIZE + 10; - /* Allocate a chunk that does not fit into a SLUB cache to trigger - * the page allocator fallback. - */ - pr_info("kmalloc pagealloc allocation: out-of-bounds to right\n"); + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_SLUB); + ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + + KUNIT_EXPECT_KASAN_FAIL(test, ptr[size + OOB_TAG_OFF] = 0); - ptr[size] = 0; kfree(ptr); } -static noinline void __init kmalloc_pagealloc_uaf(void) +static void kmalloc_pagealloc_uaf(struct kunit *test) { char *ptr; size_t size = KMALLOC_MAX_CACHE_SIZE + 10; - pr_info("kmalloc pagealloc allocation: use-after-free\n"); - ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_SLUB); + ptr = kmalloc(size, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); kfree(ptr); - ptr[0] = 0; + + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[0]); } -static noinline void __init kmalloc_pagealloc_invalid_free(void) +static void kmalloc_pagealloc_invalid_free(struct kunit *test) { char *ptr; size_t size = KMALLOC_MAX_CACHE_SIZE + 10; - pr_info("kmalloc pagealloc allocation: invalid-free\n"); + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_SLUB); + ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - kfree(ptr + 1); + KUNIT_EXPECT_KASAN_FAIL(test, kfree(ptr + 1)); } -#endif -static noinline void __init kmalloc_large_oob_right(void) +static void pagealloc_oob_right(struct kunit *test) +{ + char *ptr; + struct page *pages; + size_t order = 4; + size_t size = (1UL << (PAGE_SHIFT + order)); + + /* + * With generic KASAN page allocations have no redzones, thus + * out-of-bounds detection is not guaranteed. + * See https://bugzilla.kernel.org/show_bug.cgi?id=210503. + */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); + + pages = alloc_pages(GFP_KERNEL, order); + ptr = page_address(pages); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + + KUNIT_EXPECT_KASAN_FAIL(test, ptr[0] = ptr[size]); + free_pages((unsigned long)ptr, order); +} + +static void pagealloc_uaf(struct kunit *test) +{ + char *ptr; + struct page *pages; + size_t order = 4; + + pages = alloc_pages(GFP_KERNEL, order); + ptr = page_address(pages); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + free_pages((unsigned long)ptr, order); + + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[0]); +} + +static void kmalloc_large_oob_right(struct kunit *test) { char *ptr; size_t size = KMALLOC_MAX_CACHE_SIZE - 256; - /* Allocate a chunk that is large enough, but still fits into a slab + + /* + * Allocate a chunk that is large enough, but still fits into a slab * and does not trigger the page allocator fallback in SLUB. */ - pr_info("kmalloc large allocation: out-of-bounds to right\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - ptr[size] = 0; + KUNIT_EXPECT_KASAN_FAIL(test, ptr[size] = 0); kfree(ptr); } -static noinline void __init kmalloc_oob_krealloc_more(void) +static void krealloc_more_oob_helper(struct kunit *test, + size_t size1, size_t size2) { char *ptr1, *ptr2; - size_t size1 = 17; - size_t size2 = 19; + size_t middle; + + KUNIT_ASSERT_LT(test, size1, size2); + middle = size1 + (size2 - size1) / 2; - pr_info("out-of-bounds after krealloc more\n"); ptr1 = kmalloc(size1, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr1); + ptr2 = krealloc(ptr1, size2, GFP_KERNEL); - if (!ptr1 || !ptr2) { - pr_err("Allocation failed\n"); - kfree(ptr1); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2); + + /* All offsets up to size2 must be accessible. */ + ptr2[size1 - 1] = 'x'; + ptr2[size1] = 'x'; + ptr2[middle] = 'x'; + ptr2[size2 - 1] = 'x'; + + /* Generic mode is precise, so unaligned size2 must be inaccessible. */ + if (IS_ENABLED(CONFIG_KASAN_GENERIC)) + KUNIT_EXPECT_KASAN_FAIL(test, ptr2[size2] = 'x'); + + /* For all modes first aligned offset after size2 must be inaccessible. */ + KUNIT_EXPECT_KASAN_FAIL(test, + ptr2[round_up(size2, KASAN_GRANULE_SIZE)] = 'x'); - ptr2[size2] = 'x'; kfree(ptr2); } -static noinline void __init kmalloc_oob_krealloc_less(void) +static void krealloc_less_oob_helper(struct kunit *test, + size_t size1, size_t size2) { char *ptr1, *ptr2; - size_t size1 = 17; - size_t size2 = 15; + size_t middle; + + KUNIT_ASSERT_LT(test, size2, size1); + middle = size2 + (size1 - size2) / 2; - pr_info("out-of-bounds after krealloc less\n"); ptr1 = kmalloc(size1, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr1); + ptr2 = krealloc(ptr1, size2, GFP_KERNEL); - if (!ptr1 || !ptr2) { - pr_err("Allocation failed\n"); - kfree(ptr1); - return; - } - ptr2[size2] = 'x'; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2); + + /* Must be accessible for all modes. */ + ptr2[size2 - 1] = 'x'; + + /* Generic mode is precise, so unaligned size2 must be inaccessible. */ + if (IS_ENABLED(CONFIG_KASAN_GENERIC)) + KUNIT_EXPECT_KASAN_FAIL(test, ptr2[size2] = 'x'); + + /* For all modes first aligned offset after size2 must be inaccessible. */ + KUNIT_EXPECT_KASAN_FAIL(test, + ptr2[round_up(size2, KASAN_GRANULE_SIZE)] = 'x'); + + /* + * For all modes all size2, middle, and size1 should land in separate + * granules and thus the latter two offsets should be inaccessible. + */ + KUNIT_EXPECT_LE(test, round_up(size2, KASAN_GRANULE_SIZE), + round_down(middle, KASAN_GRANULE_SIZE)); + KUNIT_EXPECT_LE(test, round_up(middle, KASAN_GRANULE_SIZE), + round_down(size1, KASAN_GRANULE_SIZE)); + KUNIT_EXPECT_KASAN_FAIL(test, ptr2[middle] = 'x'); + KUNIT_EXPECT_KASAN_FAIL(test, ptr2[size1 - 1] = 'x'); + KUNIT_EXPECT_KASAN_FAIL(test, ptr2[size1] = 'x'); + kfree(ptr2); } -static noinline void __init kmalloc_oob_16(void) +static void krealloc_more_oob(struct kunit *test) +{ + krealloc_more_oob_helper(test, 201, 235); +} + +static void krealloc_less_oob(struct kunit *test) +{ + krealloc_less_oob_helper(test, 235, 201); +} + +static void krealloc_pagealloc_more_oob(struct kunit *test) +{ + /* page_alloc fallback in only implemented for SLUB. */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_SLUB); + + krealloc_more_oob_helper(test, KMALLOC_MAX_CACHE_SIZE + 201, + KMALLOC_MAX_CACHE_SIZE + 235); +} + +static void krealloc_pagealloc_less_oob(struct kunit *test) +{ + /* page_alloc fallback in only implemented for SLUB. */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_SLUB); + + krealloc_less_oob_helper(test, KMALLOC_MAX_CACHE_SIZE + 235, + KMALLOC_MAX_CACHE_SIZE + 201); +} + +/* + * Check that krealloc() detects a use-after-free, returns NULL, + * and doesn't unpoison the freed object. + */ +static void krealloc_uaf(struct kunit *test) +{ + char *ptr1, *ptr2; + int size1 = 201; + int size2 = 235; + + ptr1 = kmalloc(size1, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr1); + kfree(ptr1); + + KUNIT_EXPECT_KASAN_FAIL(test, ptr2 = krealloc(ptr1, size2, GFP_KERNEL)); + KUNIT_ASSERT_PTR_EQ(test, (void *)ptr2, NULL); + KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)ptr1); +} + +static void kmalloc_oob_16(struct kunit *test) { struct { u64 words[2]; } *ptr1, *ptr2; - pr_info("kmalloc out-of-bounds for 16-bytes access\n"); + /* This test is specifically crafted for the generic mode. */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); + ptr1 = kmalloc(sizeof(*ptr1) - 3, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr1); + ptr2 = kmalloc(sizeof(*ptr2), GFP_KERNEL); - if (!ptr1 || !ptr2) { - pr_err("Allocation failed\n"); - kfree(ptr1); - kfree(ptr2); - return; - } - *ptr1 = *ptr2; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2); + + KUNIT_EXPECT_KASAN_FAIL(test, *ptr1 = *ptr2); kfree(ptr1); kfree(ptr2); } -static noinline void __init kmalloc_oob_memset_2(void) +static void kmalloc_uaf_16(struct kunit *test) +{ + struct { + u64 words[2]; + } *ptr1, *ptr2; + + ptr1 = kmalloc(sizeof(*ptr1), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr1); + + ptr2 = kmalloc(sizeof(*ptr2), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2); + kfree(ptr2); + + KUNIT_EXPECT_KASAN_FAIL(test, *ptr1 = *ptr2); + kfree(ptr1); +} + +/* + * Note: in the memset tests below, the written range touches both valid and + * invalid memory. This makes sure that the instrumentation does not only check + * the starting address but the whole range. + */ + +static void kmalloc_oob_memset_2(struct kunit *test) { char *ptr; - size_t size = 8; + size_t size = 128 - KASAN_GRANULE_SIZE; - pr_info("out-of-bounds in memset2\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - memset(ptr+7, 0, 2); + OPTIMIZER_HIDE_VAR(size); + KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + size - 1, 0, 2)); kfree(ptr); } -static noinline void __init kmalloc_oob_memset_4(void) +static void kmalloc_oob_memset_4(struct kunit *test) { char *ptr; - size_t size = 8; + size_t size = 128 - KASAN_GRANULE_SIZE; - pr_info("out-of-bounds in memset4\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - memset(ptr+5, 0, 4); + OPTIMIZER_HIDE_VAR(size); + KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + size - 3, 0, 4)); kfree(ptr); } +static void kmalloc_oob_memset_8(struct kunit *test) +{ + char *ptr; + size_t size = 128 - KASAN_GRANULE_SIZE; + + ptr = kmalloc(size, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + + OPTIMIZER_HIDE_VAR(size); + KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + size - 7, 0, 8)); + kfree(ptr); +} -static noinline void __init kmalloc_oob_memset_8(void) +static void kmalloc_oob_memset_16(struct kunit *test) { char *ptr; - size_t size = 8; + size_t size = 128 - KASAN_GRANULE_SIZE; - pr_info("out-of-bounds in memset8\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - memset(ptr+1, 0, 8); + OPTIMIZER_HIDE_VAR(size); + KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + size - 15, 0, 16)); kfree(ptr); } -static noinline void __init kmalloc_oob_memset_16(void) +static void kmalloc_oob_in_memset(struct kunit *test) { char *ptr; - size_t size = 16; + size_t size = 128 - KASAN_GRANULE_SIZE; - pr_info("out-of-bounds in memset16\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - memset(ptr+1, 0, 16); + OPTIMIZER_HIDE_VAR(size); + KUNIT_EXPECT_KASAN_FAIL(test, + memset(ptr, 0, size + KASAN_GRANULE_SIZE)); kfree(ptr); } -static noinline void __init kmalloc_oob_in_memset(void) +static void kmalloc_memmove_negative_size(struct kunit *test) { char *ptr; - size_t size = 666; + size_t size = 64; + size_t invalid_size = -2; + + /* + * Hardware tag-based mode doesn't check memmove for negative size. + * As a result, this test introduces a side-effect memory corruption, + * which can result in a crash. + */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_HW_TAGS); - pr_info("out-of-bounds in memset\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - memset(ptr, 0, size+5); + memset((char *)ptr, 0, 64); + OPTIMIZER_HIDE_VAR(invalid_size); + KUNIT_EXPECT_KASAN_FAIL(test, + memmove((char *)ptr, (char *)ptr + 4, invalid_size)); kfree(ptr); } -static noinline void __init kmalloc_uaf(void) +static void kmalloc_memmove_invalid_size(struct kunit *test) +{ + char *ptr; + size_t size = 64; + volatile size_t invalid_size = size; + + ptr = kmalloc(size, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + + memset((char *)ptr, 0, 64); + KUNIT_EXPECT_KASAN_FAIL(test, + memmove((char *)ptr, (char *)ptr + 4, invalid_size)); + kfree(ptr); +} + +static void kmalloc_uaf(struct kunit *test) { char *ptr; size_t size = 10; - pr_info("use-after-free\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); kfree(ptr); - *(ptr + 8) = 'x'; + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[8]); } -static noinline void __init kmalloc_uaf_memset(void) +static void kmalloc_uaf_memset(struct kunit *test) { char *ptr; size_t size = 33; - pr_info("use-after-free in memset\n"); + /* + * Only generic KASAN uses quarantine, which is required to avoid a + * kernel memory corruption this test causes. + */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); + ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); kfree(ptr); - memset(ptr, 0, size); + KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr, 0, size)); } -static noinline void __init kmalloc_uaf2(void) +static void kmalloc_uaf2(struct kunit *test) { char *ptr1, *ptr2; size_t size = 43; + int counter = 0; - pr_info("use-after-free after another kmalloc\n"); +again: ptr1 = kmalloc(size, GFP_KERNEL); - if (!ptr1) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr1); kfree(ptr1); + ptr2 = kmalloc(size, GFP_KERNEL); - if (!ptr2) { - pr_err("Allocation failed\n"); - return; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2); + + /* + * For tag-based KASAN ptr1 and ptr2 tags might happen to be the same. + * Allow up to 16 attempts at generating different tags. + */ + if (!IS_ENABLED(CONFIG_KASAN_GENERIC) && ptr1 == ptr2 && counter++ < 16) { + kfree(ptr2); + goto again; } - ptr1[40] = 'x'; - if (ptr1 == ptr2) - pr_err("Could not detect use-after-free: ptr1 == ptr2\n"); + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr1)[40]); + KUNIT_EXPECT_PTR_NE(test, ptr1, ptr2); + kfree(ptr2); } -static noinline void __init kfree_via_page(void) +static void kfree_via_page(struct kunit *test) { char *ptr; size_t size = 8; struct page *page; unsigned long offset; - pr_info("invalid-free false positive (via page)\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); page = virt_to_page(ptr); offset = offset_in_page(ptr); kfree(page_address(page) + offset); } -static noinline void __init kfree_via_phys(void) +static void kfree_via_phys(struct kunit *test) { char *ptr; size_t size = 8; phys_addr_t phys; - pr_info("invalid-free false positive (via phys)\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); phys = virt_to_phys(ptr); kfree(phys_to_virt(phys)); } -static noinline void __init kmem_cache_oob(void) +static void kmem_cache_oob(struct kunit *test) { char *p; size_t size = 200; - struct kmem_cache *cache = kmem_cache_create("test_cache", - size, 0, - 0, NULL); - if (!cache) { - pr_err("Cache allocation failed\n"); - return; - } - pr_info("out-of-bounds in kmem_cache_alloc\n"); + struct kmem_cache *cache; + + cache = kmem_cache_create("test_cache", size, 0, 0, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + p = kmem_cache_alloc(cache, GFP_KERNEL); if (!p) { - pr_err("Allocation failed\n"); + kunit_err(test, "Allocation failed: %s\n", __func__); kmem_cache_destroy(cache); return; } - *p = p[size]; + KUNIT_EXPECT_KASAN_FAIL(test, *p = p[size + OOB_TAG_OFF]); + kmem_cache_free(cache, p); kmem_cache_destroy(cache); } -static noinline void __init memcg_accounted_kmem_cache(void) +static void kmem_cache_accounted(struct kunit *test) { int i; char *p; @@ -409,12 +654,8 @@ static noinline void __init memcg_accounted_kmem_cache(void) struct kmem_cache *cache; cache = kmem_cache_create("test_cache", size, 0, SLAB_ACCOUNT, NULL); - if (!cache) { - pr_err("Cache allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); - pr_info("allocate memcg accounted object\n"); /* * Several allocations with a delay to allow for lazy per memcg kmem * cache creation. @@ -432,136 +673,157 @@ free_cache: kmem_cache_destroy(cache); } -static char global_array[10]; - -static noinline void __init kasan_global_oob(void) +static void kmem_cache_bulk(struct kunit *test) { - volatile int i = 3; - char *p = &global_array[ARRAY_SIZE(global_array) + i]; + struct kmem_cache *cache; + size_t size = 200; + char *p[10]; + bool ret; + int i; - pr_info("out-of-bounds global variable\n"); - *(volatile char *)p; + cache = kmem_cache_create("test_cache", size, 0, 0, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + + ret = kmem_cache_alloc_bulk(cache, GFP_KERNEL, ARRAY_SIZE(p), (void **)&p); + if (!ret) { + kunit_err(test, "Allocation failed: %s\n", __func__); + kmem_cache_destroy(cache); + return; + } + + for (i = 0; i < ARRAY_SIZE(p); i++) + p[i][0] = p[i][size - 1] = 42; + + kmem_cache_free_bulk(cache, ARRAY_SIZE(p), (void **)&p); + kmem_cache_destroy(cache); } -static noinline void __init kasan_stack_oob(void) +static char global_array[10]; + +static void kasan_global_oob(struct kunit *test) { - char stack_array[10]; - volatile int i = 0; - char *p = &stack_array[ARRAY_SIZE(stack_array) + i]; + /* + * Deliberate out-of-bounds access. To prevent CONFIG_UBSAN_LOCAL_BOUNDS + * from failing here and panicking the kernel, access the array via a + * volatile pointer, which will prevent the compiler from being able to + * determine the array bounds. + * + * This access uses a volatile pointer to char (char *volatile) rather + * than the more conventional pointer to volatile char (volatile char *) + * because we want to prevent the compiler from making inferences about + * the pointer itself (i.e. its array bounds), not the data that it + * refers to. + */ + char *volatile array = global_array; + char *p = &array[ARRAY_SIZE(global_array) + 3]; - pr_info("out-of-bounds on stack\n"); - *(volatile char *)p; + /* Only generic mode instruments globals. */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); + + KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)p); } -static noinline void __init ksize_unpoisons_memory(void) +/* Check that ksize() makes the whole object accessible. */ +static void ksize_unpoisons_memory(struct kunit *test) { char *ptr; size_t size = 123, real_size; - pr_info("ksize() unpoisons the whole allocated chunk\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); real_size = ksize(ptr); - /* This access doesn't trigger an error. */ + + /* This access shouldn't trigger a KASAN report. */ ptr[size] = 'x'; - /* This one does. */ - ptr[real_size] = 'y'; + + /* This one must. */ + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[real_size]); + kfree(ptr); } -static noinline void __init copy_user_test(void) +/* + * Check that a use-after-free is detected by ksize() and via normal accesses + * after it. + */ +static void ksize_uaf(struct kunit *test) { - char *kmem; - char __user *usermem; - size_t size = 10; - int unused; - - kmem = kmalloc(size, GFP_KERNEL); - if (!kmem) - return; - - usermem = (char __user *)vm_mmap(NULL, 0, PAGE_SIZE, - PROT_READ | PROT_WRITE | PROT_EXEC, - MAP_ANONYMOUS | MAP_PRIVATE, 0); - if (IS_ERR(usermem)) { - pr_err("Failed to allocate user memory\n"); - kfree(kmem); - return; - } - - pr_info("out-of-bounds in copy_from_user()\n"); - unused = copy_from_user(kmem, usermem, size + 1); - - pr_info("out-of-bounds in copy_to_user()\n"); - unused = copy_to_user(usermem, kmem, size + 1); - - pr_info("out-of-bounds in __copy_from_user()\n"); - unused = __copy_from_user(kmem, usermem, size + 1); + char *ptr; + int size = 128 - KASAN_GRANULE_SIZE; - pr_info("out-of-bounds in __copy_to_user()\n"); - unused = __copy_to_user(usermem, kmem, size + 1); + ptr = kmalloc(size, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + kfree(ptr); - pr_info("out-of-bounds in __copy_from_user_inatomic()\n"); - unused = __copy_from_user_inatomic(kmem, usermem, size + 1); + KUNIT_EXPECT_KASAN_FAIL(test, ksize(ptr)); + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[0]); + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[size]); +} - pr_info("out-of-bounds in __copy_to_user_inatomic()\n"); - unused = __copy_to_user_inatomic(usermem, kmem, size + 1); +static void kasan_stack_oob(struct kunit *test) +{ + char stack_array[10]; + /* See comment in kasan_global_oob. */ + char *volatile array = stack_array; + char *p = &array[ARRAY_SIZE(stack_array) + OOB_TAG_OFF]; - pr_info("out-of-bounds in strncpy_from_user()\n"); - unused = strncpy_from_user(kmem, usermem, size + 1); + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_STACK); - vm_munmap((unsigned long)usermem, PAGE_SIZE); - kfree(kmem); + KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)p); } -static noinline void __init kasan_alloca_oob_left(void) +static void kasan_alloca_oob_left(struct kunit *test) { volatile int i = 10; char alloca_array[i]; - char *p = alloca_array - 1; + /* See comment in kasan_global_oob. */ + char *volatile array = alloca_array; + char *p = array - 1; + + /* Only generic mode instruments dynamic allocas. */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_STACK); - pr_info("out-of-bounds to left on alloca\n"); - *(volatile char *)p; + KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)p); } -static noinline void __init kasan_alloca_oob_right(void) +static void kasan_alloca_oob_right(struct kunit *test) { volatile int i = 10; char alloca_array[i]; - char *p = alloca_array + i; + /* See comment in kasan_global_oob. */ + char *volatile array = alloca_array; + char *p = array + i; + + /* Only generic mode instruments dynamic allocas. */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_STACK); - pr_info("out-of-bounds to right on alloca\n"); - *(volatile char *)p; + KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)p); } -static noinline void __init kmem_cache_double_free(void) +static void kmem_cache_double_free(struct kunit *test) { char *p; size_t size = 200; struct kmem_cache *cache; cache = kmem_cache_create("test_cache", size, 0, 0, NULL); - if (!cache) { - pr_err("Cache allocation failed\n"); - return; - } - pr_info("double-free on heap object\n"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + p = kmem_cache_alloc(cache, GFP_KERNEL); if (!p) { - pr_err("Allocation failed\n"); + kunit_err(test, "Allocation failed: %s\n", __func__); kmem_cache_destroy(cache); return; } kmem_cache_free(cache, p); - kmem_cache_free(cache, p); + KUNIT_EXPECT_KASAN_FAIL(test, kmem_cache_free(cache, p)); kmem_cache_destroy(cache); } -static noinline void __init kmem_cache_invalid_free(void) +static void kmem_cache_invalid_free(struct kunit *test) { char *p; size_t size = 200; @@ -569,20 +831,17 @@ static noinline void __init kmem_cache_invalid_free(void) cache = kmem_cache_create("test_cache", size, 0, SLAB_TYPESAFE_BY_RCU, NULL); - if (!cache) { - pr_err("Cache allocation failed\n"); - return; - } - pr_info("invalid-free of heap object\n"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + p = kmem_cache_alloc(cache, GFP_KERNEL); if (!p) { - pr_err("Allocation failed\n"); + kunit_err(test, "Allocation failed: %s\n", __func__); kmem_cache_destroy(cache); return; } - /* Trigger invalid free, the object doesn't get freed */ - kmem_cache_free(cache, p + 1); + /* Trigger invalid free, the object doesn't get freed. */ + KUNIT_EXPECT_KASAN_FAIL(test, kmem_cache_free(cache, p + 1)); /* * Properly free the object to prevent the "Objects remaining in @@ -593,45 +852,66 @@ static noinline void __init kmem_cache_invalid_free(void) kmem_cache_destroy(cache); } -static noinline void __init kasan_memchr(void) +static void kasan_memchr(struct kunit *test) { char *ptr; size_t size = 24; - pr_info("out-of-bounds in memchr\n"); + /* + * str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT. + * See https://bugzilla.kernel.org/show_bug.cgi?id=206337 for details. + */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_AMD_MEM_ENCRYPT); + + if (OOB_TAG_OFF) + size = round_up(size, OOB_TAG_OFF); + ptr = kmalloc(size, GFP_KERNEL | __GFP_ZERO); - if (!ptr) - return; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + + KUNIT_EXPECT_KASAN_FAIL(test, + kasan_ptr_result = memchr(ptr, '1', size + 1)); - memchr(ptr, '1', size + 1); kfree(ptr); } -static noinline void __init kasan_memcmp(void) +static void kasan_memcmp(struct kunit *test) { char *ptr; size_t size = 24; int arr[9]; - pr_info("out-of-bounds in memcmp\n"); - ptr = kmalloc(size, GFP_KERNEL | __GFP_ZERO); - if (!ptr) - return; + /* + * str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT. + * See https://bugzilla.kernel.org/show_bug.cgi?id=206337 for details. + */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_AMD_MEM_ENCRYPT); + if (OOB_TAG_OFF) + size = round_up(size, OOB_TAG_OFF); + + ptr = kmalloc(size, GFP_KERNEL | __GFP_ZERO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); memset(arr, 0, sizeof(arr)); - memcmp(ptr, arr, size+1); + + KUNIT_EXPECT_KASAN_FAIL(test, + kasan_int_result = memcmp(ptr, arr, size+1)); kfree(ptr); } -static noinline void __init kasan_strings(void) +static void kasan_strings(struct kunit *test) { char *ptr; size_t size = 24; - pr_info("use-after-free in strchr\n"); + /* + * str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT. + * See https://bugzilla.kernel.org/show_bug.cgi?id=206337 for details. + */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_AMD_MEM_ENCRYPT); + ptr = kmalloc(size, GFP_KERNEL | __GFP_ZERO); - if (!ptr) - return; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); kfree(ptr); @@ -642,188 +922,272 @@ static noinline void __init kasan_strings(void) * will likely point to zeroed byte. */ ptr += 16; - strchr(ptr, '1'); + KUNIT_EXPECT_KASAN_FAIL(test, kasan_ptr_result = strchr(ptr, '1')); + + KUNIT_EXPECT_KASAN_FAIL(test, kasan_ptr_result = strrchr(ptr, '1')); + + KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = strcmp(ptr, "2")); + + KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = strncmp(ptr, "2", 1)); - pr_info("use-after-free in strrchr\n"); - strrchr(ptr, '1'); + KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = strlen(ptr)); - pr_info("use-after-free in strcmp\n"); - strcmp(ptr, "2"); + KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = strnlen(ptr, 1)); +} - pr_info("use-after-free in strncmp\n"); - strncmp(ptr, "2", 1); +static void kasan_bitops_modify(struct kunit *test, int nr, void *addr) +{ + KUNIT_EXPECT_KASAN_FAIL(test, set_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, __set_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, clear_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, __clear_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, clear_bit_unlock(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, __clear_bit_unlock(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, change_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, __change_bit(nr, addr)); +} - pr_info("use-after-free in strlen\n"); - strlen(ptr); +static void kasan_bitops_test_and_modify(struct kunit *test, int nr, void *addr) +{ + KUNIT_EXPECT_KASAN_FAIL(test, test_and_set_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, __test_and_set_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, test_and_set_bit_lock(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, test_and_clear_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, __test_and_clear_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, test_and_change_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, __test_and_change_bit(nr, addr)); + KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = test_bit(nr, addr)); - pr_info("use-after-free in strnlen\n"); - strnlen(ptr, 1); +#if defined(clear_bit_unlock_is_negative_byte) + KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = + clear_bit_unlock_is_negative_byte(nr, addr)); +#endif } -static noinline void __init kasan_bitops(void) +static void kasan_bitops_generic(struct kunit *test) { + long *bits; + + /* This test is specifically crafted for the generic mode. */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); + /* - * Allocate 1 more byte, which causes kzalloc to round up to 16-bytes; + * Allocate 1 more byte, which causes kzalloc to round up to 16 bytes; * this way we do not actually corrupt other memory. */ - long *bits = kzalloc(sizeof(*bits) + 1, GFP_KERNEL); - if (!bits) - return; + bits = kzalloc(sizeof(*bits) + 1, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bits); /* * Below calls try to access bit within allocated memory; however, the * below accesses are still out-of-bounds, since bitops are defined to * operate on the whole long the bit is in. */ - pr_info("out-of-bounds in set_bit\n"); - set_bit(BITS_PER_LONG, bits); - - pr_info("out-of-bounds in __set_bit\n"); - __set_bit(BITS_PER_LONG, bits); - - pr_info("out-of-bounds in clear_bit\n"); - clear_bit(BITS_PER_LONG, bits); - - pr_info("out-of-bounds in __clear_bit\n"); - __clear_bit(BITS_PER_LONG, bits); - - pr_info("out-of-bounds in clear_bit_unlock\n"); - clear_bit_unlock(BITS_PER_LONG, bits); - - pr_info("out-of-bounds in __clear_bit_unlock\n"); - __clear_bit_unlock(BITS_PER_LONG, bits); - - pr_info("out-of-bounds in change_bit\n"); - change_bit(BITS_PER_LONG, bits); - - pr_info("out-of-bounds in __change_bit\n"); - __change_bit(BITS_PER_LONG, bits); + kasan_bitops_modify(test, BITS_PER_LONG, bits); /* * Below calls try to access bit beyond allocated memory. */ - pr_info("out-of-bounds in test_and_set_bit\n"); - test_and_set_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); - - pr_info("out-of-bounds in __test_and_set_bit\n"); - __test_and_set_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); - - pr_info("out-of-bounds in test_and_set_bit_lock\n"); - test_and_set_bit_lock(BITS_PER_LONG + BITS_PER_BYTE, bits); + kasan_bitops_test_and_modify(test, BITS_PER_LONG + BITS_PER_BYTE, bits); - pr_info("out-of-bounds in test_and_clear_bit\n"); - test_and_clear_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + kfree(bits); +} - pr_info("out-of-bounds in __test_and_clear_bit\n"); - __test_and_clear_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); +static void kasan_bitops_tags(struct kunit *test) +{ + long *bits; - pr_info("out-of-bounds in test_and_change_bit\n"); - test_and_change_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + /* This test is specifically crafted for tag-based modes. */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); - pr_info("out-of-bounds in __test_and_change_bit\n"); - __test_and_change_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + /* kmalloc-64 cache will be used and the last 16 bytes will be the redzone. */ + bits = kzalloc(48, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bits); - pr_info("out-of-bounds in test_bit\n"); - (void)test_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + /* Do the accesses past the 48 allocated bytes, but within the redone. */ + kasan_bitops_modify(test, BITS_PER_LONG, (void *)bits + 48); + kasan_bitops_test_and_modify(test, BITS_PER_LONG + BITS_PER_BYTE, (void *)bits + 48); -#if defined(clear_bit_unlock_is_negative_byte) - pr_info("out-of-bounds in clear_bit_unlock_is_negative_byte\n"); - clear_bit_unlock_is_negative_byte(BITS_PER_LONG + BITS_PER_BYTE, bits); -#endif kfree(bits); } -static noinline void __init kmalloc_double_kzfree(void) +static void kmalloc_double_kzfree(struct kunit *test) { char *ptr; size_t size = 16; - pr_info("double-free (kzfree)\n"); ptr = kmalloc(size, GFP_KERNEL); - if (!ptr) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - kzfree(ptr); - kzfree(ptr); + kfree_sensitive(ptr); + KUNIT_EXPECT_KASAN_FAIL(test, kfree_sensitive(ptr)); } -#ifdef CONFIG_KASAN_VMALLOC -static noinline void __init vmalloc_oob(void) +static void vmalloc_oob(struct kunit *test) { void *area; - pr_info("vmalloc out-of-bounds\n"); + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_VMALLOC); /* * We have to be careful not to hit the guard page. * The MMU will catch that and crash us. */ area = vmalloc(3000); - if (!area) { - pr_err("Allocation failed\n"); - return; - } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, area); - ((volatile char *)area)[3100]; + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)area)[3100]); vfree(area); } -#else -static void __init vmalloc_oob(void) {} -#endif -static int __init kmalloc_tests_init(void) +/* + * Check that the assigned pointer tag falls within the [KASAN_TAG_MIN, + * KASAN_TAG_KERNEL) range (note: excluding the match-all tag) for tag-based + * modes. + */ +static void match_all_not_assigned(struct kunit *test) { - /* - * Temporarily enable multi-shot mode. Otherwise, we'd only get a - * report for the first case. - */ - bool multishot = kasan_save_enable_multi_shot(); - - kmalloc_oob_right(); - kmalloc_oob_left(); - kmalloc_node_oob_right(); -#ifdef CONFIG_SLUB - kmalloc_pagealloc_oob_right(); - kmalloc_pagealloc_uaf(); - kmalloc_pagealloc_invalid_free(); -#endif - kmalloc_large_oob_right(); - kmalloc_oob_krealloc_more(); - kmalloc_oob_krealloc_less(); - kmalloc_oob_16(); - kmalloc_oob_in_memset(); - kmalloc_oob_memset_2(); - kmalloc_oob_memset_4(); - kmalloc_oob_memset_8(); - kmalloc_oob_memset_16(); - kmalloc_uaf(); - kmalloc_uaf_memset(); - kmalloc_uaf2(); - kfree_via_page(); - kfree_via_phys(); - kmem_cache_oob(); - memcg_accounted_kmem_cache(); - kasan_stack_oob(); - kasan_global_oob(); - kasan_alloca_oob_left(); - kasan_alloca_oob_right(); - ksize_unpoisons_memory(); - copy_user_test(); - kmem_cache_double_free(); - kmem_cache_invalid_free(); - kasan_memchr(); - kasan_memcmp(); - kasan_strings(); - kasan_bitops(); - kmalloc_double_kzfree(); - vmalloc_oob(); + char *ptr; + struct page *pages; + int i, size, order; + + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); + + for (i = 0; i < 256; i++) { + size = (get_random_int() % 1024) + 1; + ptr = kmalloc(size, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + KUNIT_EXPECT_GE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_MIN); + KUNIT_EXPECT_LT(test, (u8)get_tag(ptr), (u8)KASAN_TAG_KERNEL); + kfree(ptr); + } - kasan_restore_multi_shot(multishot); + for (i = 0; i < 256; i++) { + order = (get_random_int() % 4) + 1; + pages = alloc_pages(GFP_KERNEL, order); + ptr = page_address(pages); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + KUNIT_EXPECT_GE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_MIN); + KUNIT_EXPECT_LT(test, (u8)get_tag(ptr), (u8)KASAN_TAG_KERNEL); + free_pages((unsigned long)ptr, order); + } +} + +/* Check that 0xff works as a match-all pointer tag for tag-based modes. */ +static void match_all_ptr_tag(struct kunit *test) +{ + char *ptr; + u8 tag; + + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); + + ptr = kmalloc(128, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + + /* Backup the assigned tag. */ + tag = get_tag(ptr); + KUNIT_EXPECT_NE(test, tag, (u8)KASAN_TAG_KERNEL); + + /* Reset the tag to 0xff.*/ + ptr = set_tag(ptr, KASAN_TAG_KERNEL); + + /* This access shouldn't trigger a KASAN report. */ + *ptr = 0; + + /* Recover the pointer tag and free. */ + ptr = set_tag(ptr, tag); + kfree(ptr); +} + +/* Check that there are no match-all memory tags for tag-based modes. */ +static void match_all_mem_tag(struct kunit *test) +{ + char *ptr; + int tag; + + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); + + ptr = kmalloc(128, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + KUNIT_EXPECT_NE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_KERNEL); + + /* For each possible tag value not matching the pointer tag. */ + for (tag = KASAN_TAG_MIN; tag <= KASAN_TAG_KERNEL; tag++) { + if (tag == get_tag(ptr)) + continue; - return -EAGAIN; + /* Mark the first memory granule with the chosen memory tag. */ + kasan_poison(ptr, KASAN_GRANULE_SIZE, (u8)tag, false); + + /* This access must cause a KASAN report. */ + KUNIT_EXPECT_KASAN_FAIL(test, *ptr = 0); + } + + /* Recover the memory tag and free. */ + kasan_poison(ptr, KASAN_GRANULE_SIZE, get_tag(ptr), false); + kfree(ptr); } -module_init(kmalloc_tests_init); +static struct kunit_case kasan_kunit_test_cases[] = { + KUNIT_CASE(kmalloc_oob_right), + KUNIT_CASE(kmalloc_oob_left), + KUNIT_CASE(kmalloc_node_oob_right), + KUNIT_CASE(kmalloc_pagealloc_oob_right), + KUNIT_CASE(kmalloc_pagealloc_uaf), + KUNIT_CASE(kmalloc_pagealloc_invalid_free), + KUNIT_CASE(pagealloc_oob_right), + KUNIT_CASE(pagealloc_uaf), + KUNIT_CASE(kmalloc_large_oob_right), + KUNIT_CASE(krealloc_more_oob), + KUNIT_CASE(krealloc_less_oob), + KUNIT_CASE(krealloc_pagealloc_more_oob), + KUNIT_CASE(krealloc_pagealloc_less_oob), + KUNIT_CASE(krealloc_uaf), + KUNIT_CASE(kmalloc_oob_16), + KUNIT_CASE(kmalloc_uaf_16), + KUNIT_CASE(kmalloc_oob_in_memset), + KUNIT_CASE(kmalloc_oob_memset_2), + KUNIT_CASE(kmalloc_oob_memset_4), + KUNIT_CASE(kmalloc_oob_memset_8), + KUNIT_CASE(kmalloc_oob_memset_16), + KUNIT_CASE(kmalloc_memmove_negative_size), + KUNIT_CASE(kmalloc_memmove_invalid_size), + KUNIT_CASE(kmalloc_uaf), + KUNIT_CASE(kmalloc_uaf_memset), + KUNIT_CASE(kmalloc_uaf2), + KUNIT_CASE(kfree_via_page), + KUNIT_CASE(kfree_via_phys), + KUNIT_CASE(kmem_cache_oob), + KUNIT_CASE(kmem_cache_accounted), + KUNIT_CASE(kmem_cache_bulk), + KUNIT_CASE(kasan_global_oob), + KUNIT_CASE(kasan_stack_oob), + KUNIT_CASE(kasan_alloca_oob_left), + KUNIT_CASE(kasan_alloca_oob_right), + KUNIT_CASE(ksize_unpoisons_memory), + KUNIT_CASE(ksize_uaf), + KUNIT_CASE(kmem_cache_double_free), + KUNIT_CASE(kmem_cache_invalid_free), + KUNIT_CASE(kasan_memchr), + KUNIT_CASE(kasan_memcmp), + KUNIT_CASE(kasan_strings), + KUNIT_CASE(kasan_bitops_generic), + KUNIT_CASE(kasan_bitops_tags), + KUNIT_CASE(kmalloc_double_kzfree), + KUNIT_CASE(vmalloc_oob), + KUNIT_CASE(match_all_not_assigned), + KUNIT_CASE(match_all_ptr_tag), + KUNIT_CASE(match_all_mem_tag), + {} +}; + +static struct kunit_suite kasan_kunit_test_suite = { + .name = "kasan", + .init = kasan_test_init, + .test_cases = kasan_kunit_test_cases, + .exit = kasan_test_exit, +}; + +kunit_test_suite(kasan_kunit_test_suite); + MODULE_LICENSE("GPL"); diff --git a/lib/test_kasan_module.c b/lib/test_kasan_module.c new file mode 100644 index 000000000000..b112cbc835e9 --- /dev/null +++ b/lib/test_kasan_module.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * + * Copyright (c) 2014 Samsung Electronics Co., Ltd. + * Author: Andrey Ryabinin <a.ryabinin@samsung.com> + */ + +#define pr_fmt(fmt) "kasan test: %s " fmt, __func__ + +#include <linux/mman.h> +#include <linux/module.h> +#include <linux/printk.h> +#include <linux/slab.h> +#include <linux/uaccess.h> + +#include "../mm/kasan/kasan.h" + +static noinline void __init copy_user_test(void) +{ + char *kmem; + char __user *usermem; + size_t size = 128 - KASAN_GRANULE_SIZE; + int __maybe_unused unused; + + kmem = kmalloc(size, GFP_KERNEL); + if (!kmem) + return; + + usermem = (char __user *)vm_mmap(NULL, 0, PAGE_SIZE, + PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_ANONYMOUS | MAP_PRIVATE, 0); + if (IS_ERR(usermem)) { + pr_err("Failed to allocate user memory\n"); + kfree(kmem); + return; + } + + OPTIMIZER_HIDE_VAR(size); + + pr_info("out-of-bounds in copy_from_user()\n"); + unused = copy_from_user(kmem, usermem, size + 1); + + pr_info("out-of-bounds in copy_to_user()\n"); + unused = copy_to_user(usermem, kmem, size + 1); + + pr_info("out-of-bounds in __copy_from_user()\n"); + unused = __copy_from_user(kmem, usermem, size + 1); + + pr_info("out-of-bounds in __copy_to_user()\n"); + unused = __copy_to_user(usermem, kmem, size + 1); + + pr_info("out-of-bounds in __copy_from_user_inatomic()\n"); + unused = __copy_from_user_inatomic(kmem, usermem, size + 1); + + pr_info("out-of-bounds in __copy_to_user_inatomic()\n"); + unused = __copy_to_user_inatomic(usermem, kmem, size + 1); + + pr_info("out-of-bounds in strncpy_from_user()\n"); + unused = strncpy_from_user(kmem, usermem, size + 1); + + vm_munmap((unsigned long)usermem, PAGE_SIZE); + kfree(kmem); +} + +static struct kasan_rcu_info { + int i; + struct rcu_head rcu; +} *global_rcu_ptr; + +static noinline void __init kasan_rcu_reclaim(struct rcu_head *rp) +{ + struct kasan_rcu_info *fp = container_of(rp, + struct kasan_rcu_info, rcu); + + kfree(fp); + ((volatile struct kasan_rcu_info *)fp)->i; +} + +static noinline void __init kasan_rcu_uaf(void) +{ + struct kasan_rcu_info *ptr; + + pr_info("use-after-free in kasan_rcu_reclaim\n"); + ptr = kmalloc(sizeof(struct kasan_rcu_info), GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + global_rcu_ptr = rcu_dereference_protected(ptr, NULL); + call_rcu(&global_rcu_ptr->rcu, kasan_rcu_reclaim); +} + +static noinline void __init kasan_workqueue_work(struct work_struct *work) +{ + kfree(work); +} + +static noinline void __init kasan_workqueue_uaf(void) +{ + struct workqueue_struct *workqueue; + struct work_struct *work; + + workqueue = create_workqueue("kasan_wq_test"); + if (!workqueue) { + pr_err("Allocation failed\n"); + return; + } + work = kmalloc(sizeof(struct work_struct), GFP_KERNEL); + if (!work) { + pr_err("Allocation failed\n"); + return; + } + + INIT_WORK(work, kasan_workqueue_work); + queue_work(workqueue, work); + destroy_workqueue(workqueue); + + pr_info("use-after-free on workqueue\n"); + ((volatile struct work_struct *)work)->data; +} + +static int __init test_kasan_module_init(void) +{ + /* + * Temporarily enable multi-shot mode. Otherwise, KASAN would only + * report the first detected bug and panic the kernel if panic_on_warn + * is enabled. + */ + bool multishot = kasan_save_enable_multi_shot(); + + copy_user_test(); + kasan_rcu_uaf(); + kasan_workqueue_uaf(); + + kasan_restore_multi_shot(multishot); + return -EAGAIN; +} + +module_init(test_kasan_module_init); +MODULE_LICENSE("GPL"); diff --git a/lib/test_kmod.c b/lib/test_kmod.c index 9cf77628fc91..ce1589391413 100644 --- a/lib/test_kmod.c +++ b/lib/test_kmod.c @@ -204,7 +204,7 @@ static void test_kmod_put_module(struct kmod_test_device_info *info) case TEST_KMOD_DRIVER: break; case TEST_KMOD_FS_TYPE: - if (info && info->fs_sync && info->fs_sync->owner) + if (info->fs_sync && info->fs_sync->owner) module_put(info->fs_sync->owner); break; default: @@ -286,7 +286,7 @@ static int tally_work_test(struct kmod_test_device_info *info) * If this ran it means *all* tasks were created fine and we * are now just collecting results. * - * Only propagate errors, do not override with a subsequent sucess case. + * Only propagate errors, do not override with a subsequent success case. */ static void tally_up_work(struct kmod_test_device *test_dev) { @@ -543,7 +543,7 @@ static int trigger_config_run(struct kmod_test_device *test_dev) * wrong with the setup of the test. If the test setup went fine * then userspace must just check the result of config->test_result. * One issue with relying on the return from a call in the kernel - * is if the kernel returns a possitive value using this trigger + * is if the kernel returns a positive value using this trigger * will not return the value to userspace, it would be lost. * * By not relying on capturing the return value of tests we are using @@ -585,7 +585,7 @@ trigger_config_store(struct device *dev, * Note: any return > 0 will be treated as success * and the error value will not be available to userspace. * Do not rely on trying to send to userspace a test value - * return value as possitive return errors will be lost. + * return value as positive return errors will be lost. */ if (WARN_ON(ret > 0)) return -EINVAL; @@ -745,7 +745,7 @@ static int trigger_config_run_type(struct kmod_test_device *test_dev, break; case TEST_KMOD_FS_TYPE: kfree_const(config->test_fs); - config->test_driver = NULL; + config->test_fs = NULL; copied = config_copy_test_fs(config, test_str, strlen(test_str)); break; @@ -877,20 +877,17 @@ static int test_dev_config_update_uint_sync(struct kmod_test_device *test_dev, int (*test_sync)(struct kmod_test_device *test_dev)) { int ret; - unsigned long new; + unsigned int val; unsigned int old_val; - ret = kstrtoul(buf, 10, &new); + ret = kstrtouint(buf, 10, &val); if (ret) return ret; - if (new > UINT_MAX) - return -EINVAL; - mutex_lock(&test_dev->config_mutex); old_val = *config; - *(unsigned int *)config = new; + *(unsigned int *)config = val; ret = test_sync(test_dev); if (ret) { @@ -914,18 +911,18 @@ static int test_dev_config_update_uint_range(struct kmod_test_device *test_dev, unsigned int min, unsigned int max) { + unsigned int val; int ret; - unsigned long new; - ret = kstrtoul(buf, 10, &new); + ret = kstrtouint(buf, 10, &val); if (ret) return ret; - if (new < min || new > max) + if (val < min || val > max) return -EINVAL; mutex_lock(&test_dev->config_mutex); - *config = new; + *config = val; mutex_unlock(&test_dev->config_mutex); /* Always return full write size even if we didn't consume all */ @@ -936,18 +933,15 @@ static int test_dev_config_update_int(struct kmod_test_device *test_dev, const char *buf, size_t size, int *config) { + int val; int ret; - long new; - ret = kstrtol(buf, 10, &new); + ret = kstrtoint(buf, 10, &val); if (ret) return ret; - if (new < INT_MIN || new > INT_MAX) - return -EINVAL; - mutex_lock(&test_dev->config_mutex); - *config = new; + *config = val; mutex_unlock(&test_dev->config_mutex); /* Always return full write size even if we didn't consume all */ return size; diff --git a/lib/test_kprobes.c b/lib/test_kprobes.c new file mode 100644 index 000000000000..a5edc2ebc947 --- /dev/null +++ b/lib/test_kprobes.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * test_kprobes.c - simple sanity test for *probes + * + * Copyright IBM Corp. 2008 + */ + +#include <linux/kernel.h> +#include <linux/kprobes.h> +#include <linux/random.h> +#include <kunit/test.h> + +#define div_factor 3 + +static u32 rand1, preh_val, posth_val; +static u32 (*target)(u32 value); +static u32 (*target2)(u32 value); +static struct kunit *current_test; + +static unsigned long (*internal_target)(void); +static unsigned long (*stacktrace_target)(void); +static unsigned long (*stacktrace_driver)(void); +static unsigned long target_return_address[2]; + +static noinline u32 kprobe_target(u32 value) +{ + return (value / div_factor); +} + +static int kp_pre_handler(struct kprobe *p, struct pt_regs *regs) +{ + KUNIT_EXPECT_FALSE(current_test, preemptible()); + preh_val = (rand1 / div_factor); + return 0; +} + +static void kp_post_handler(struct kprobe *p, struct pt_regs *regs, + unsigned long flags) +{ + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, preh_val, (rand1 / div_factor)); + posth_val = preh_val + div_factor; +} + +static struct kprobe kp = { + .symbol_name = "kprobe_target", + .pre_handler = kp_pre_handler, + .post_handler = kp_post_handler +}; + +static void test_kprobe(struct kunit *test) +{ + current_test = test; + KUNIT_EXPECT_EQ(test, 0, register_kprobe(&kp)); + target(rand1); + unregister_kprobe(&kp); + KUNIT_EXPECT_NE(test, 0, preh_val); + KUNIT_EXPECT_NE(test, 0, posth_val); +} + +static noinline u32 kprobe_target2(u32 value) +{ + return (value / div_factor) + 1; +} + +static noinline unsigned long kprobe_stacktrace_internal_target(void) +{ + if (!target_return_address[0]) + target_return_address[0] = (unsigned long)__builtin_return_address(0); + return target_return_address[0]; +} + +static noinline unsigned long kprobe_stacktrace_target(void) +{ + if (!target_return_address[1]) + target_return_address[1] = (unsigned long)__builtin_return_address(0); + + if (internal_target) + internal_target(); + + return target_return_address[1]; +} + +static noinline unsigned long kprobe_stacktrace_driver(void) +{ + if (stacktrace_target) + stacktrace_target(); + + /* This is for preventing inlining the function */ + return (unsigned long)__builtin_return_address(0); +} + +static int kp_pre_handler2(struct kprobe *p, struct pt_regs *regs) +{ + preh_val = (rand1 / div_factor) + 1; + return 0; +} + +static void kp_post_handler2(struct kprobe *p, struct pt_regs *regs, + unsigned long flags) +{ + KUNIT_EXPECT_EQ(current_test, preh_val, (rand1 / div_factor) + 1); + posth_val = preh_val + div_factor; +} + +static struct kprobe kp2 = { + .symbol_name = "kprobe_target2", + .pre_handler = kp_pre_handler2, + .post_handler = kp_post_handler2 +}; + +static void test_kprobes(struct kunit *test) +{ + struct kprobe *kps[2] = {&kp, &kp2}; + + current_test = test; + + /* addr and flags should be cleard for reusing kprobe. */ + kp.addr = NULL; + kp.flags = 0; + + KUNIT_EXPECT_EQ(test, 0, register_kprobes(kps, 2)); + preh_val = 0; + posth_val = 0; + target(rand1); + + KUNIT_EXPECT_NE(test, 0, preh_val); + KUNIT_EXPECT_NE(test, 0, posth_val); + + preh_val = 0; + posth_val = 0; + target2(rand1); + + KUNIT_EXPECT_NE(test, 0, preh_val); + KUNIT_EXPECT_NE(test, 0, posth_val); + unregister_kprobes(kps, 2); +} + +#ifdef CONFIG_KRETPROBES +static u32 krph_val; + +static int entry_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + KUNIT_EXPECT_FALSE(current_test, preemptible()); + krph_val = (rand1 / div_factor); + return 0; +} + +static int return_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + unsigned long ret = regs_return_value(regs); + + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, ret, rand1 / div_factor); + KUNIT_EXPECT_NE(current_test, krph_val, 0); + krph_val = rand1; + return 0; +} + +static struct kretprobe rp = { + .handler = return_handler, + .entry_handler = entry_handler, + .kp.symbol_name = "kprobe_target" +}; + +static void test_kretprobe(struct kunit *test) +{ + current_test = test; + KUNIT_EXPECT_EQ(test, 0, register_kretprobe(&rp)); + target(rand1); + unregister_kretprobe(&rp); + KUNIT_EXPECT_EQ(test, krph_val, rand1); +} + +static int return_handler2(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + unsigned long ret = regs_return_value(regs); + + KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor) + 1); + KUNIT_EXPECT_NE(current_test, krph_val, 0); + krph_val = rand1; + return 0; +} + +static struct kretprobe rp2 = { + .handler = return_handler2, + .entry_handler = entry_handler, + .kp.symbol_name = "kprobe_target2" +}; + +static void test_kretprobes(struct kunit *test) +{ + struct kretprobe *rps[2] = {&rp, &rp2}; + + current_test = test; + /* addr and flags should be cleard for reusing kprobe. */ + rp.kp.addr = NULL; + rp.kp.flags = 0; + KUNIT_EXPECT_EQ(test, 0, register_kretprobes(rps, 2)); + + krph_val = 0; + target(rand1); + KUNIT_EXPECT_EQ(test, krph_val, rand1); + + krph_val = 0; + target2(rand1); + KUNIT_EXPECT_EQ(test, krph_val, rand1); + unregister_kretprobes(rps, 2); +} + +#ifdef CONFIG_ARCH_CORRECT_STACKTRACE_ON_KRETPROBE +#define STACK_BUF_SIZE 16 +static unsigned long stack_buf[STACK_BUF_SIZE]; + +static int stacktrace_return_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + unsigned long retval = regs_return_value(regs); + int i, ret; + + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, retval, target_return_address[1]); + + /* + * Test stacktrace inside the kretprobe handler, this will involves + * kretprobe trampoline, but must include correct return address + * of the target function. + */ + ret = stack_trace_save(stack_buf, STACK_BUF_SIZE, 0); + KUNIT_EXPECT_NE(current_test, ret, 0); + + for (i = 0; i < ret; i++) { + if (stack_buf[i] == target_return_address[1]) + break; + } + KUNIT_EXPECT_NE(current_test, i, ret); + +#if !IS_MODULE(CONFIG_KPROBES_SANITY_TEST) + /* + * Test stacktrace from pt_regs at the return address. Thus the stack + * trace must start from the target return address. + */ + ret = stack_trace_save_regs(regs, stack_buf, STACK_BUF_SIZE, 0); + KUNIT_EXPECT_NE(current_test, ret, 0); + KUNIT_EXPECT_EQ(current_test, stack_buf[0], target_return_address[1]); +#endif + + return 0; +} + +static struct kretprobe rp3 = { + .handler = stacktrace_return_handler, + .kp.symbol_name = "kprobe_stacktrace_target" +}; + +static void test_stacktrace_on_kretprobe(struct kunit *test) +{ + unsigned long myretaddr = (unsigned long)__builtin_return_address(0); + + current_test = test; + rp3.kp.addr = NULL; + rp3.kp.flags = 0; + + /* + * Run the stacktrace_driver() to record correct return address in + * stacktrace_target() and ensure stacktrace_driver() call is not + * inlined by checking the return address of stacktrace_driver() + * and the return address of this function is different. + */ + KUNIT_ASSERT_NE(test, myretaddr, stacktrace_driver()); + + KUNIT_ASSERT_EQ(test, 0, register_kretprobe(&rp3)); + KUNIT_ASSERT_NE(test, myretaddr, stacktrace_driver()); + unregister_kretprobe(&rp3); +} + +static int stacktrace_internal_return_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + unsigned long retval = regs_return_value(regs); + int i, ret; + + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, retval, target_return_address[0]); + + /* + * Test stacktrace inside the kretprobe handler for nested case. + * The unwinder will find the kretprobe_trampoline address on the + * return address, and kretprobe must solve that. + */ + ret = stack_trace_save(stack_buf, STACK_BUF_SIZE, 0); + KUNIT_EXPECT_NE(current_test, ret, 0); + + for (i = 0; i < ret - 1; i++) { + if (stack_buf[i] == target_return_address[0]) { + KUNIT_EXPECT_EQ(current_test, stack_buf[i + 1], target_return_address[1]); + break; + } + } + KUNIT_EXPECT_NE(current_test, i, ret); + +#if !IS_MODULE(CONFIG_KPROBES_SANITY_TEST) + /* Ditto for the regs version. */ + ret = stack_trace_save_regs(regs, stack_buf, STACK_BUF_SIZE, 0); + KUNIT_EXPECT_NE(current_test, ret, 0); + KUNIT_EXPECT_EQ(current_test, stack_buf[0], target_return_address[0]); + KUNIT_EXPECT_EQ(current_test, stack_buf[1], target_return_address[1]); +#endif + + return 0; +} + +static struct kretprobe rp4 = { + .handler = stacktrace_internal_return_handler, + .kp.symbol_name = "kprobe_stacktrace_internal_target" +}; + +static void test_stacktrace_on_nested_kretprobe(struct kunit *test) +{ + unsigned long myretaddr = (unsigned long)__builtin_return_address(0); + struct kretprobe *rps[2] = {&rp3, &rp4}; + + current_test = test; + rp3.kp.addr = NULL; + rp3.kp.flags = 0; + + //KUNIT_ASSERT_NE(test, myretaddr, stacktrace_driver()); + + KUNIT_ASSERT_EQ(test, 0, register_kretprobes(rps, 2)); + KUNIT_ASSERT_NE(test, myretaddr, stacktrace_driver()); + unregister_kretprobes(rps, 2); +} +#endif /* CONFIG_ARCH_CORRECT_STACKTRACE_ON_KRETPROBE */ + +#endif /* CONFIG_KRETPROBES */ + +static int kprobes_test_init(struct kunit *test) +{ + target = kprobe_target; + target2 = kprobe_target2; + stacktrace_target = kprobe_stacktrace_target; + internal_target = kprobe_stacktrace_internal_target; + stacktrace_driver = kprobe_stacktrace_driver; + + do { + rand1 = prandom_u32(); + } while (rand1 <= div_factor); + return 0; +} + +static struct kunit_case kprobes_testcases[] = { + KUNIT_CASE(test_kprobe), + KUNIT_CASE(test_kprobes), +#ifdef CONFIG_KRETPROBES + KUNIT_CASE(test_kretprobe), + KUNIT_CASE(test_kretprobes), +#ifdef CONFIG_ARCH_CORRECT_STACKTRACE_ON_KRETPROBE + KUNIT_CASE(test_stacktrace_on_kretprobe), + KUNIT_CASE(test_stacktrace_on_nested_kretprobe), +#endif +#endif + {} +}; + +static struct kunit_suite kprobes_test_suite = { + .name = "kprobes_test", + .init = kprobes_test_init, + .test_cases = kprobes_testcases, +}; + +kunit_test_suites(&kprobes_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/lib/test_linear_ranges.c b/lib/test_linear_ranges.c new file mode 100644 index 000000000000..676e0b8abcdd --- /dev/null +++ b/lib/test_linear_ranges.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for the linear_ranges helper. + * + * Copyright (C) 2020, ROHM Semiconductors. + * Author: Matti Vaittinen <matti.vaittien@fi.rohmeurope.com> + */ +#include <kunit/test.h> + +#include <linux/linear_range.h> + +/* First things first. I deeply dislike unit-tests. I have seen all the hell + * breaking loose when people who think the unit tests are "the silver bullet" + * to kill bugs get to decide how a company should implement testing strategy... + * + * Believe me, it may get _really_ ridiculous. It is tempting to think that + * walking through all the possible execution branches will nail down 100% of + * bugs. This may lead to ideas about demands to get certain % of "test + * coverage" - measured as line coverage. And that is one of the worst things + * you can do. + * + * Ask people to provide line coverage and they do. I've seen clever tools + * which generate test cases to test the existing functions - and by default + * these tools expect code to be correct and just generate checks which are + * passing when ran against current code-base. Run this generator and you'll get + * tests that do not test code is correct but just verify nothing changes. + * Problem is that testing working code is pointless. And if it is not + * working, your test must not assume it is working. You won't catch any bugs + * by such tests. What you can do is to generate a huge amount of tests. + * Especially if you were are asked to proivde 100% line-coverage x_x. So what + * does these tests - which are not finding any bugs now - do? + * + * They add inertia to every future development. I think it was Terry Pratchet + * who wrote someone having same impact as thick syrup has to chronometre. + * Excessive amount of unit-tests have this effect to development. If you do + * actually find _any_ bug from code in such environment and try fixing it... + * ...chances are you also need to fix the test cases. In sunny day you fix one + * test. But I've done refactoring which resulted 500+ broken tests (which had + * really zero value other than proving to managers that we do do "quality")... + * + * After this being said - there are situations where UTs can be handy. If you + * have algorithms which take some input and should produce output - then you + * can implement few, carefully selected simple UT-cases which test this. I've + * previously used this for example for netlink and device-tree data parsing + * functions. Feed some data examples to functions and verify the output is as + * expected. I am not covering all the cases but I will see the logic should be + * working. + * + * Here we also do some minor testing. I don't want to go through all branches + * or test more or less obvious things - but I want to see the main logic is + * working. And I definitely don't want to add 500+ test cases that break when + * some simple fix is done x_x. So - let's only add few, well selected tests + * which ensure as much logic is good as possible. + */ + +/* + * Test Range 1: + * selectors: 2 3 4 5 6 + * values (5): 10 20 30 40 50 + * + * Test Range 2: + * selectors: 7 8 9 10 + * values (4): 100 150 200 250 + */ + +#define RANGE1_MIN 10 +#define RANGE1_MIN_SEL 2 +#define RANGE1_STEP 10 + +/* 2, 3, 4, 5, 6 */ +static const unsigned int range1_sels[] = { RANGE1_MIN_SEL, RANGE1_MIN_SEL + 1, + RANGE1_MIN_SEL + 2, + RANGE1_MIN_SEL + 3, + RANGE1_MIN_SEL + 4 }; +/* 10, 20, 30, 40, 50 */ +static const unsigned int range1_vals[] = { RANGE1_MIN, RANGE1_MIN + + RANGE1_STEP, + RANGE1_MIN + RANGE1_STEP * 2, + RANGE1_MIN + RANGE1_STEP * 3, + RANGE1_MIN + RANGE1_STEP * 4 }; + +#define RANGE2_MIN 100 +#define RANGE2_MIN_SEL 7 +#define RANGE2_STEP 50 + +/* 7, 8, 9, 10 */ +static const unsigned int range2_sels[] = { RANGE2_MIN_SEL, RANGE2_MIN_SEL + 1, + RANGE2_MIN_SEL + 2, + RANGE2_MIN_SEL + 3 }; +/* 100, 150, 200, 250 */ +static const unsigned int range2_vals[] = { RANGE2_MIN, RANGE2_MIN + + RANGE2_STEP, + RANGE2_MIN + RANGE2_STEP * 2, + RANGE2_MIN + RANGE2_STEP * 3 }; + +#define RANGE1_NUM_VALS (ARRAY_SIZE(range1_vals)) +#define RANGE2_NUM_VALS (ARRAY_SIZE(range2_vals)) +#define RANGE_NUM_VALS (RANGE1_NUM_VALS + RANGE2_NUM_VALS) + +#define RANGE1_MAX_SEL (RANGE1_MIN_SEL + RANGE1_NUM_VALS - 1) +#define RANGE1_MAX_VAL (range1_vals[RANGE1_NUM_VALS - 1]) + +#define RANGE2_MAX_SEL (RANGE2_MIN_SEL + RANGE2_NUM_VALS - 1) +#define RANGE2_MAX_VAL (range2_vals[RANGE2_NUM_VALS - 1]) + +#define SMALLEST_SEL RANGE1_MIN_SEL +#define SMALLEST_VAL RANGE1_MIN + +static struct linear_range testr[] = { + { + .min = RANGE1_MIN, + .min_sel = RANGE1_MIN_SEL, + .max_sel = RANGE1_MAX_SEL, + .step = RANGE1_STEP, + }, { + .min = RANGE2_MIN, + .min_sel = RANGE2_MIN_SEL, + .max_sel = RANGE2_MAX_SEL, + .step = RANGE2_STEP + }, +}; + +static void range_test_get_value(struct kunit *test) +{ + int ret, i; + unsigned int sel, val; + + for (i = 0; i < RANGE1_NUM_VALS; i++) { + sel = range1_sels[i]; + ret = linear_range_get_value_array(&testr[0], 2, sel, &val); + KUNIT_EXPECT_EQ(test, 0, ret); + KUNIT_EXPECT_EQ(test, val, range1_vals[i]); + } + for (i = 0; i < RANGE2_NUM_VALS; i++) { + sel = range2_sels[i]; + ret = linear_range_get_value_array(&testr[0], 2, sel, &val); + KUNIT_EXPECT_EQ(test, 0, ret); + KUNIT_EXPECT_EQ(test, val, range2_vals[i]); + } + ret = linear_range_get_value_array(&testr[0], 2, sel + 1, &val); + KUNIT_EXPECT_NE(test, 0, ret); +} + +static void range_test_get_selector_high(struct kunit *test) +{ + int ret, i; + unsigned int sel; + bool found; + + for (i = 0; i < RANGE1_NUM_VALS; i++) { + ret = linear_range_get_selector_high(&testr[0], range1_vals[i], + &sel, &found); + KUNIT_EXPECT_EQ(test, 0, ret); + KUNIT_EXPECT_EQ(test, sel, range1_sels[i]); + KUNIT_EXPECT_TRUE(test, found); + } + + ret = linear_range_get_selector_high(&testr[0], RANGE1_MAX_VAL + 1, + &sel, &found); + KUNIT_EXPECT_LE(test, ret, 0); + + ret = linear_range_get_selector_high(&testr[0], RANGE1_MIN - 1, + &sel, &found); + KUNIT_EXPECT_EQ(test, 0, ret); + KUNIT_EXPECT_FALSE(test, found); + KUNIT_EXPECT_EQ(test, sel, range1_sels[0]); +} + +static void range_test_get_value_amount(struct kunit *test) +{ + int ret; + + ret = linear_range_values_in_range_array(&testr[0], 2); + KUNIT_EXPECT_EQ(test, (int)RANGE_NUM_VALS, ret); +} + +static void range_test_get_selector_low(struct kunit *test) +{ + int i, ret; + unsigned int sel; + bool found; + + for (i = 0; i < RANGE1_NUM_VALS; i++) { + ret = linear_range_get_selector_low_array(&testr[0], 2, + range1_vals[i], &sel, + &found); + KUNIT_EXPECT_EQ(test, 0, ret); + KUNIT_EXPECT_EQ(test, sel, range1_sels[i]); + KUNIT_EXPECT_TRUE(test, found); + } + for (i = 0; i < RANGE2_NUM_VALS; i++) { + ret = linear_range_get_selector_low_array(&testr[0], 2, + range2_vals[i], &sel, + &found); + KUNIT_EXPECT_EQ(test, 0, ret); + KUNIT_EXPECT_EQ(test, sel, range2_sels[i]); + KUNIT_EXPECT_TRUE(test, found); + } + + /* + * Seek value greater than range max => get_selector_*_low should + * return Ok - but set found to false as value is not in range + */ + ret = linear_range_get_selector_low_array(&testr[0], 2, + range2_vals[RANGE2_NUM_VALS - 1] + 1, + &sel, &found); + + KUNIT_EXPECT_EQ(test, 0, ret); + KUNIT_EXPECT_EQ(test, sel, range2_sels[RANGE2_NUM_VALS - 1]); + KUNIT_EXPECT_FALSE(test, found); +} + +static struct kunit_case range_test_cases[] = { + KUNIT_CASE(range_test_get_value_amount), + KUNIT_CASE(range_test_get_selector_high), + KUNIT_CASE(range_test_get_selector_low), + KUNIT_CASE(range_test_get_value), + {}, +}; + +static struct kunit_suite range_test_module = { + .name = "linear-ranges-test", + .test_cases = range_test_cases, +}; + +kunit_test_suites(&range_test_module); + +MODULE_LICENSE("GPL"); diff --git a/lib/test_list_sort.c b/lib/test_list_sort.c index 1f017d3b610e..ade7a1ea0c8e 100644 --- a/lib/test_list_sort.c +++ b/lib/test_list_sort.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -#define pr_fmt(fmt) "list_sort_test: " fmt +#include <kunit/test.h> #include <linux/kernel.h> #include <linux/list_sort.h> @@ -23,67 +23,52 @@ struct debug_el { struct list_head list; unsigned int poison2; int value; - unsigned serial; + unsigned int serial; }; -/* Array, containing pointers to all elements in the test list */ -static struct debug_el **elts __initdata; - -static int __init check(struct debug_el *ela, struct debug_el *elb) +static void check(struct kunit *test, struct debug_el *ela, struct debug_el *elb) { - if (ela->serial >= TEST_LIST_LEN) { - pr_err("error: incorrect serial %d\n", ela->serial); - return -EINVAL; - } - if (elb->serial >= TEST_LIST_LEN) { - pr_err("error: incorrect serial %d\n", elb->serial); - return -EINVAL; - } - if (elts[ela->serial] != ela || elts[elb->serial] != elb) { - pr_err("error: phantom element\n"); - return -EINVAL; - } - if (ela->poison1 != TEST_POISON1 || ela->poison2 != TEST_POISON2) { - pr_err("error: bad poison: %#x/%#x\n", - ela->poison1, ela->poison2); - return -EINVAL; - } - if (elb->poison1 != TEST_POISON1 || elb->poison2 != TEST_POISON2) { - pr_err("error: bad poison: %#x/%#x\n", - elb->poison1, elb->poison2); - return -EINVAL; - } - return 0; + struct debug_el **elts = test->priv; + + KUNIT_EXPECT_LT_MSG(test, ela->serial, (unsigned int)TEST_LIST_LEN, "incorrect serial"); + KUNIT_EXPECT_LT_MSG(test, elb->serial, (unsigned int)TEST_LIST_LEN, "incorrect serial"); + + KUNIT_EXPECT_PTR_EQ_MSG(test, elts[ela->serial], ela, "phantom element"); + KUNIT_EXPECT_PTR_EQ_MSG(test, elts[elb->serial], elb, "phantom element"); + + KUNIT_EXPECT_EQ_MSG(test, ela->poison1, TEST_POISON1, "bad poison"); + KUNIT_EXPECT_EQ_MSG(test, ela->poison2, TEST_POISON2, "bad poison"); + + KUNIT_EXPECT_EQ_MSG(test, elb->poison1, TEST_POISON1, "bad poison"); + KUNIT_EXPECT_EQ_MSG(test, elb->poison2, TEST_POISON2, "bad poison"); } -static int __init cmp(void *priv, struct list_head *a, struct list_head *b) +/* `priv` is the test pointer so check() can fail the test if the list is invalid. */ +static int cmp(void *priv, const struct list_head *a, const struct list_head *b) { struct debug_el *ela, *elb; ela = container_of(a, struct debug_el, list); elb = container_of(b, struct debug_el, list); - check(ela, elb); + check(priv, ela, elb); return ela->value - elb->value; } -static int __init list_sort_test(void) +static void list_sort_test(struct kunit *test) { - int i, count = 1, err = -ENOMEM; - struct debug_el *el; + int i, count = 1; + struct debug_el *el, **elts; struct list_head *cur; LIST_HEAD(head); - pr_debug("start testing list_sort()\n"); - - elts = kcalloc(TEST_LIST_LEN, sizeof(*elts), GFP_KERNEL); - if (!elts) - return err; + elts = kunit_kcalloc(test, TEST_LIST_LEN, sizeof(*elts), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, elts); + test->priv = elts; for (i = 0; i < TEST_LIST_LEN; i++) { - el = kmalloc(sizeof(*el), GFP_KERNEL); - if (!el) - goto exit; + el = kunit_kmalloc(test, sizeof(*el), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, el); /* force some equivalencies */ el->value = prandom_u32() % (TEST_LIST_LEN / 3); @@ -94,55 +79,44 @@ static int __init list_sort_test(void) list_add_tail(&el->list, &head); } - list_sort(NULL, &head, cmp); + list_sort(test, &head, cmp); - err = -EINVAL; for (cur = head.next; cur->next != &head; cur = cur->next) { struct debug_el *el1; int cmp_result; - if (cur->next->prev != cur) { - pr_err("error: list is corrupted\n"); - goto exit; - } + KUNIT_ASSERT_PTR_EQ_MSG(test, cur->next->prev, cur, + "list is corrupted"); - cmp_result = cmp(NULL, cur, cur->next); - if (cmp_result > 0) { - pr_err("error: list is not sorted\n"); - goto exit; - } + cmp_result = cmp(test, cur, cur->next); + KUNIT_ASSERT_LE_MSG(test, cmp_result, 0, "list is not sorted"); el = container_of(cur, struct debug_el, list); el1 = container_of(cur->next, struct debug_el, list); - if (cmp_result == 0 && el->serial >= el1->serial) { - pr_err("error: order of equivalent elements not " - "preserved\n"); - goto exit; + if (cmp_result == 0) { + KUNIT_ASSERT_LE_MSG(test, el->serial, el1->serial, + "order of equivalent elements not preserved"); } - if (check(el, el1)) { - pr_err("error: element check failed\n"); - goto exit; - } + check(test, el, el1); count++; } - if (head.prev != cur) { - pr_err("error: list is corrupted\n"); - goto exit; - } + KUNIT_EXPECT_PTR_EQ_MSG(test, head.prev, cur, "list is corrupted"); + KUNIT_EXPECT_EQ_MSG(test, count, TEST_LIST_LEN, + "list length changed after sorting!"); +} - if (count != TEST_LIST_LEN) { - pr_err("error: bad list length %d", count); - goto exit; - } +static struct kunit_case list_sort_cases[] = { + KUNIT_CASE(list_sort_test), + {} +}; + +static struct kunit_suite list_sort_suite = { + .name = "list_sort", + .test_cases = list_sort_cases, +}; + +kunit_test_suites(&list_sort_suite); - err = 0; -exit: - for (i = 0; i < TEST_LIST_LEN; i++) - kfree(elts[i]); - kfree(elts); - return err; -} -module_init(list_sort_test); MODULE_LICENSE("GPL"); diff --git a/lib/test_lockup.c b/lib/test_lockup.c new file mode 100644 index 000000000000..906b598740a7 --- /dev/null +++ b/lib/test_lockup.c @@ -0,0 +1,615 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test module to generate lockups + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/delay.h> +#include <linux/sched.h> +#include <linux/sched/signal.h> +#include <linux/sched/clock.h> +#include <linux/cpu.h> +#include <linux/nmi.h> +#include <linux/mm.h> +#include <linux/uaccess.h> +#include <linux/file.h> + +static unsigned int time_secs; +module_param(time_secs, uint, 0600); +MODULE_PARM_DESC(time_secs, "lockup time in seconds, default 0"); + +static unsigned int time_nsecs; +module_param(time_nsecs, uint, 0600); +MODULE_PARM_DESC(time_nsecs, "nanoseconds part of lockup time, default 0"); + +static unsigned int cooldown_secs; +module_param(cooldown_secs, uint, 0600); +MODULE_PARM_DESC(cooldown_secs, "cooldown time between iterations in seconds, default 0"); + +static unsigned int cooldown_nsecs; +module_param(cooldown_nsecs, uint, 0600); +MODULE_PARM_DESC(cooldown_nsecs, "nanoseconds part of cooldown, default 0"); + +static unsigned int iterations = 1; +module_param(iterations, uint, 0600); +MODULE_PARM_DESC(iterations, "lockup iterations, default 1"); + +static bool all_cpus; +module_param(all_cpus, bool, 0400); +MODULE_PARM_DESC(all_cpus, "trigger lockup at all cpus at once"); + +static int wait_state; +static char *state = "R"; +module_param(state, charp, 0400); +MODULE_PARM_DESC(state, "wait in 'R' running (default), 'D' uninterruptible, 'K' killable, 'S' interruptible state"); + +static bool use_hrtimer; +module_param(use_hrtimer, bool, 0400); +MODULE_PARM_DESC(use_hrtimer, "use high-resolution timer for sleeping"); + +static bool iowait; +module_param(iowait, bool, 0400); +MODULE_PARM_DESC(iowait, "account sleep time as iowait"); + +static bool lock_read; +module_param(lock_read, bool, 0400); +MODULE_PARM_DESC(lock_read, "lock read-write locks for read"); + +static bool lock_single; +module_param(lock_single, bool, 0400); +MODULE_PARM_DESC(lock_single, "acquire locks only at one cpu"); + +static bool reacquire_locks; +module_param(reacquire_locks, bool, 0400); +MODULE_PARM_DESC(reacquire_locks, "release and reacquire locks/irq/preempt between iterations"); + +static bool touch_softlockup; +module_param(touch_softlockup, bool, 0600); +MODULE_PARM_DESC(touch_softlockup, "touch soft-lockup watchdog between iterations"); + +static bool touch_hardlockup; +module_param(touch_hardlockup, bool, 0600); +MODULE_PARM_DESC(touch_hardlockup, "touch hard-lockup watchdog between iterations"); + +static bool call_cond_resched; +module_param(call_cond_resched, bool, 0600); +MODULE_PARM_DESC(call_cond_resched, "call cond_resched() between iterations"); + +static bool measure_lock_wait; +module_param(measure_lock_wait, bool, 0400); +MODULE_PARM_DESC(measure_lock_wait, "measure lock wait time"); + +static unsigned long lock_wait_threshold = ULONG_MAX; +module_param(lock_wait_threshold, ulong, 0400); +MODULE_PARM_DESC(lock_wait_threshold, "print lock wait time longer than this in nanoseconds, default off"); + +static bool test_disable_irq; +module_param_named(disable_irq, test_disable_irq, bool, 0400); +MODULE_PARM_DESC(disable_irq, "disable interrupts: generate hard-lockups"); + +static bool disable_softirq; +module_param(disable_softirq, bool, 0400); +MODULE_PARM_DESC(disable_softirq, "disable bottom-half irq handlers"); + +static bool disable_preempt; +module_param(disable_preempt, bool, 0400); +MODULE_PARM_DESC(disable_preempt, "disable preemption: generate soft-lockups"); + +static bool lock_rcu; +module_param(lock_rcu, bool, 0400); +MODULE_PARM_DESC(lock_rcu, "grab rcu_read_lock: generate rcu stalls"); + +static bool lock_mmap_sem; +module_param(lock_mmap_sem, bool, 0400); +MODULE_PARM_DESC(lock_mmap_sem, "lock mm->mmap_lock: block procfs interfaces"); + +static unsigned long lock_rwsem_ptr; +module_param_unsafe(lock_rwsem_ptr, ulong, 0400); +MODULE_PARM_DESC(lock_rwsem_ptr, "lock rw_semaphore at address"); + +static unsigned long lock_mutex_ptr; +module_param_unsafe(lock_mutex_ptr, ulong, 0400); +MODULE_PARM_DESC(lock_mutex_ptr, "lock mutex at address"); + +static unsigned long lock_spinlock_ptr; +module_param_unsafe(lock_spinlock_ptr, ulong, 0400); +MODULE_PARM_DESC(lock_spinlock_ptr, "lock spinlock at address"); + +static unsigned long lock_rwlock_ptr; +module_param_unsafe(lock_rwlock_ptr, ulong, 0400); +MODULE_PARM_DESC(lock_rwlock_ptr, "lock rwlock at address"); + +static unsigned int alloc_pages_nr; +module_param_unsafe(alloc_pages_nr, uint, 0600); +MODULE_PARM_DESC(alloc_pages_nr, "allocate and free pages under locks"); + +static unsigned int alloc_pages_order; +module_param(alloc_pages_order, uint, 0400); +MODULE_PARM_DESC(alloc_pages_order, "page order to allocate"); + +static gfp_t alloc_pages_gfp = GFP_KERNEL; +module_param_unsafe(alloc_pages_gfp, uint, 0400); +MODULE_PARM_DESC(alloc_pages_gfp, "allocate pages with this gfp_mask, default GFP_KERNEL"); + +static bool alloc_pages_atomic; +module_param(alloc_pages_atomic, bool, 0400); +MODULE_PARM_DESC(alloc_pages_atomic, "allocate pages with GFP_ATOMIC"); + +static bool reallocate_pages; +module_param(reallocate_pages, bool, 0400); +MODULE_PARM_DESC(reallocate_pages, "free and allocate pages between iterations"); + +struct file *test_file; +static struct inode *test_inode; +static char test_file_path[256]; +module_param_string(file_path, test_file_path, sizeof(test_file_path), 0400); +MODULE_PARM_DESC(file_path, "file path to test"); + +static bool test_lock_inode; +module_param_named(lock_inode, test_lock_inode, bool, 0400); +MODULE_PARM_DESC(lock_inode, "lock file -> inode -> i_rwsem"); + +static bool test_lock_mapping; +module_param_named(lock_mapping, test_lock_mapping, bool, 0400); +MODULE_PARM_DESC(lock_mapping, "lock file -> mapping -> i_mmap_rwsem"); + +static bool test_lock_sb_umount; +module_param_named(lock_sb_umount, test_lock_sb_umount, bool, 0400); +MODULE_PARM_DESC(lock_sb_umount, "lock file -> sb -> s_umount"); + +static atomic_t alloc_pages_failed = ATOMIC_INIT(0); + +static atomic64_t max_lock_wait = ATOMIC64_INIT(0); + +static struct task_struct *main_task; +static int master_cpu; + +static void test_lock(bool master, bool verbose) +{ + u64 wait_start; + + if (measure_lock_wait) + wait_start = local_clock(); + + if (lock_mutex_ptr && master) { + if (verbose) + pr_notice("lock mutex %ps\n", (void *)lock_mutex_ptr); + mutex_lock((struct mutex *)lock_mutex_ptr); + } + + if (lock_rwsem_ptr && master) { + if (verbose) + pr_notice("lock rw_semaphore %ps\n", + (void *)lock_rwsem_ptr); + if (lock_read) + down_read((struct rw_semaphore *)lock_rwsem_ptr); + else + down_write((struct rw_semaphore *)lock_rwsem_ptr); + } + + if (lock_mmap_sem && master) { + if (verbose) + pr_notice("lock mmap_lock pid=%d\n", main_task->pid); + if (lock_read) + mmap_read_lock(main_task->mm); + else + mmap_write_lock(main_task->mm); + } + + if (test_disable_irq) + local_irq_disable(); + + if (disable_softirq) + local_bh_disable(); + + if (disable_preempt) + preempt_disable(); + + if (lock_rcu) + rcu_read_lock(); + + if (lock_spinlock_ptr && master) { + if (verbose) + pr_notice("lock spinlock %ps\n", + (void *)lock_spinlock_ptr); + spin_lock((spinlock_t *)lock_spinlock_ptr); + } + + if (lock_rwlock_ptr && master) { + if (verbose) + pr_notice("lock rwlock %ps\n", + (void *)lock_rwlock_ptr); + if (lock_read) + read_lock((rwlock_t *)lock_rwlock_ptr); + else + write_lock((rwlock_t *)lock_rwlock_ptr); + } + + if (measure_lock_wait) { + s64 cur_wait = local_clock() - wait_start; + s64 max_wait = atomic64_read(&max_lock_wait); + + do { + if (cur_wait < max_wait) + break; + max_wait = atomic64_cmpxchg(&max_lock_wait, + max_wait, cur_wait); + } while (max_wait != cur_wait); + + if (cur_wait > lock_wait_threshold) + pr_notice_ratelimited("lock wait %lld ns\n", cur_wait); + } +} + +static void test_unlock(bool master, bool verbose) +{ + if (lock_rwlock_ptr && master) { + if (lock_read) + read_unlock((rwlock_t *)lock_rwlock_ptr); + else + write_unlock((rwlock_t *)lock_rwlock_ptr); + if (verbose) + pr_notice("unlock rwlock %ps\n", + (void *)lock_rwlock_ptr); + } + + if (lock_spinlock_ptr && master) { + spin_unlock((spinlock_t *)lock_spinlock_ptr); + if (verbose) + pr_notice("unlock spinlock %ps\n", + (void *)lock_spinlock_ptr); + } + + if (lock_rcu) + rcu_read_unlock(); + + if (disable_preempt) + preempt_enable(); + + if (disable_softirq) + local_bh_enable(); + + if (test_disable_irq) + local_irq_enable(); + + if (lock_mmap_sem && master) { + if (lock_read) + mmap_read_unlock(main_task->mm); + else + mmap_write_unlock(main_task->mm); + if (verbose) + pr_notice("unlock mmap_lock pid=%d\n", main_task->pid); + } + + if (lock_rwsem_ptr && master) { + if (lock_read) + up_read((struct rw_semaphore *)lock_rwsem_ptr); + else + up_write((struct rw_semaphore *)lock_rwsem_ptr); + if (verbose) + pr_notice("unlock rw_semaphore %ps\n", + (void *)lock_rwsem_ptr); + } + + if (lock_mutex_ptr && master) { + mutex_unlock((struct mutex *)lock_mutex_ptr); + if (verbose) + pr_notice("unlock mutex %ps\n", + (void *)lock_mutex_ptr); + } +} + +static void test_alloc_pages(struct list_head *pages) +{ + struct page *page; + unsigned int i; + + for (i = 0; i < alloc_pages_nr; i++) { + page = alloc_pages(alloc_pages_gfp, alloc_pages_order); + if (!page) { + atomic_inc(&alloc_pages_failed); + break; + } + list_add(&page->lru, pages); + } +} + +static void test_free_pages(struct list_head *pages) +{ + struct page *page, *next; + + list_for_each_entry_safe(page, next, pages, lru) + __free_pages(page, alloc_pages_order); + INIT_LIST_HEAD(pages); +} + +static void test_wait(unsigned int secs, unsigned int nsecs) +{ + if (wait_state == TASK_RUNNING) { + if (secs) + mdelay(secs * MSEC_PER_SEC); + if (nsecs) + ndelay(nsecs); + return; + } + + __set_current_state(wait_state); + if (use_hrtimer) { + ktime_t time; + + time = ns_to_ktime((u64)secs * NSEC_PER_SEC + nsecs); + schedule_hrtimeout(&time, HRTIMER_MODE_REL); + } else { + schedule_timeout(secs * HZ + nsecs_to_jiffies(nsecs)); + } +} + +static void test_lockup(bool master) +{ + u64 lockup_start = local_clock(); + unsigned int iter = 0; + LIST_HEAD(pages); + + pr_notice("Start on CPU%d\n", raw_smp_processor_id()); + + test_lock(master, true); + + test_alloc_pages(&pages); + + while (iter++ < iterations && !signal_pending(main_task)) { + + if (iowait) + current->in_iowait = 1; + + test_wait(time_secs, time_nsecs); + + if (iowait) + current->in_iowait = 0; + + if (reallocate_pages) + test_free_pages(&pages); + + if (reacquire_locks) + test_unlock(master, false); + + if (touch_softlockup) + touch_softlockup_watchdog(); + + if (touch_hardlockup) + touch_nmi_watchdog(); + + if (call_cond_resched) + cond_resched(); + + test_wait(cooldown_secs, cooldown_nsecs); + + if (reacquire_locks) + test_lock(master, false); + + if (reallocate_pages) + test_alloc_pages(&pages); + } + + pr_notice("Finish on CPU%d in %lld ns\n", raw_smp_processor_id(), + local_clock() - lockup_start); + + test_free_pages(&pages); + + test_unlock(master, true); +} + +static DEFINE_PER_CPU(struct work_struct, test_works); + +static void test_work_fn(struct work_struct *work) +{ + test_lockup(!lock_single || + work == per_cpu_ptr(&test_works, master_cpu)); +} + +static bool test_kernel_ptr(unsigned long addr, int size) +{ + void *ptr = (void *)addr; + char buf; + + if (!addr) + return false; + + /* should be at least readable kernel address */ + if (access_ok(ptr, 1) || + access_ok(ptr + size - 1, 1) || + get_kernel_nofault(buf, ptr) || + get_kernel_nofault(buf, ptr + size - 1)) { + pr_err("invalid kernel ptr: %#lx\n", addr); + return true; + } + + return false; +} + +static bool __maybe_unused test_magic(unsigned long addr, int offset, + unsigned int expected) +{ + void *ptr = (void *)addr + offset; + unsigned int magic = 0; + + if (!addr) + return false; + + if (get_kernel_nofault(magic, ptr) || magic != expected) { + pr_err("invalid magic at %#lx + %#x = %#x, expected %#x\n", + addr, offset, magic, expected); + return true; + } + + return false; +} + +static int __init test_lockup_init(void) +{ + u64 test_start = local_clock(); + + main_task = current; + + switch (state[0]) { + case 'S': + wait_state = TASK_INTERRUPTIBLE; + break; + case 'D': + wait_state = TASK_UNINTERRUPTIBLE; + break; + case 'K': + wait_state = TASK_KILLABLE; + break; + case 'R': + wait_state = TASK_RUNNING; + break; + default: + pr_err("unknown state=%s\n", state); + return -EINVAL; + } + + if (alloc_pages_atomic) + alloc_pages_gfp = GFP_ATOMIC; + + if (test_kernel_ptr(lock_spinlock_ptr, sizeof(spinlock_t)) || + test_kernel_ptr(lock_rwlock_ptr, sizeof(rwlock_t)) || + test_kernel_ptr(lock_mutex_ptr, sizeof(struct mutex)) || + test_kernel_ptr(lock_rwsem_ptr, sizeof(struct rw_semaphore))) + return -EINVAL; + +#ifdef CONFIG_DEBUG_SPINLOCK +#ifdef CONFIG_PREEMPT_RT + if (test_magic(lock_spinlock_ptr, + offsetof(spinlock_t, lock.wait_lock.magic), + SPINLOCK_MAGIC) || + test_magic(lock_rwlock_ptr, + offsetof(rwlock_t, rwbase.rtmutex.wait_lock.magic), + SPINLOCK_MAGIC) || + test_magic(lock_mutex_ptr, + offsetof(struct mutex, rtmutex.wait_lock.magic), + SPINLOCK_MAGIC) || + test_magic(lock_rwsem_ptr, + offsetof(struct rw_semaphore, rwbase.rtmutex.wait_lock.magic), + SPINLOCK_MAGIC)) + return -EINVAL; +#else + if (test_magic(lock_spinlock_ptr, + offsetof(spinlock_t, rlock.magic), + SPINLOCK_MAGIC) || + test_magic(lock_rwlock_ptr, + offsetof(rwlock_t, magic), + RWLOCK_MAGIC) || + test_magic(lock_mutex_ptr, + offsetof(struct mutex, wait_lock.magic), + SPINLOCK_MAGIC) || + test_magic(lock_rwsem_ptr, + offsetof(struct rw_semaphore, wait_lock.magic), + SPINLOCK_MAGIC)) + return -EINVAL; +#endif +#endif + + if ((wait_state != TASK_RUNNING || + (call_cond_resched && !reacquire_locks) || + (alloc_pages_nr && gfpflags_allow_blocking(alloc_pages_gfp))) && + (test_disable_irq || disable_softirq || disable_preempt || + lock_rcu || lock_spinlock_ptr || lock_rwlock_ptr)) { + pr_err("refuse to sleep in atomic context\n"); + return -EINVAL; + } + + if (lock_mmap_sem && !main_task->mm) { + pr_err("no mm to lock mmap_lock\n"); + return -EINVAL; + } + + if (test_file_path[0]) { + test_file = filp_open(test_file_path, O_RDONLY, 0); + if (IS_ERR(test_file)) { + pr_err("failed to open %s: %ld\n", test_file_path, PTR_ERR(test_file)); + return PTR_ERR(test_file); + } + test_inode = file_inode(test_file); + } else if (test_lock_inode || + test_lock_mapping || + test_lock_sb_umount) { + pr_err("no file to lock\n"); + return -EINVAL; + } + + if (test_lock_inode && test_inode) + lock_rwsem_ptr = (unsigned long)&test_inode->i_rwsem; + + if (test_lock_mapping && test_file && test_file->f_mapping) + lock_rwsem_ptr = (unsigned long)&test_file->f_mapping->i_mmap_rwsem; + + if (test_lock_sb_umount && test_inode) + lock_rwsem_ptr = (unsigned long)&test_inode->i_sb->s_umount; + + pr_notice("START pid=%d time=%u +%u ns cooldown=%u +%u ns iterations=%u state=%s %s%s%s%s%s%s%s%s%s%s%s\n", + main_task->pid, time_secs, time_nsecs, + cooldown_secs, cooldown_nsecs, iterations, state, + all_cpus ? "all_cpus " : "", + iowait ? "iowait " : "", + test_disable_irq ? "disable_irq " : "", + disable_softirq ? "disable_softirq " : "", + disable_preempt ? "disable_preempt " : "", + lock_rcu ? "lock_rcu " : "", + lock_read ? "lock_read " : "", + touch_softlockup ? "touch_softlockup " : "", + touch_hardlockup ? "touch_hardlockup " : "", + call_cond_resched ? "call_cond_resched " : "", + reacquire_locks ? "reacquire_locks " : ""); + + if (alloc_pages_nr) + pr_notice("ALLOCATE PAGES nr=%u order=%u gfp=%pGg %s\n", + alloc_pages_nr, alloc_pages_order, &alloc_pages_gfp, + reallocate_pages ? "reallocate_pages " : ""); + + if (all_cpus) { + unsigned int cpu; + + cpus_read_lock(); + + preempt_disable(); + master_cpu = smp_processor_id(); + for_each_online_cpu(cpu) { + INIT_WORK(per_cpu_ptr(&test_works, cpu), test_work_fn); + queue_work_on(cpu, system_highpri_wq, + per_cpu_ptr(&test_works, cpu)); + } + preempt_enable(); + + for_each_online_cpu(cpu) + flush_work(per_cpu_ptr(&test_works, cpu)); + + cpus_read_unlock(); + } else { + test_lockup(true); + } + + if (measure_lock_wait) + pr_notice("Maximum lock wait: %lld ns\n", + atomic64_read(&max_lock_wait)); + + if (alloc_pages_nr) + pr_notice("Page allocation failed %u times\n", + atomic_read(&alloc_pages_failed)); + + pr_notice("FINISH in %llu ns\n", local_clock() - test_start); + + if (test_file) + fput(test_file); + + if (signal_pending(main_task)) + return -EINTR; + + return -EAGAIN; +} +module_init(test_lockup_init); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Konstantin Khlebnikov <khlebnikov@yandex-team.ru>"); +MODULE_DESCRIPTION("Test module to generate lockups"); diff --git a/lib/test_min_heap.c b/lib/test_min_heap.c new file mode 100644 index 000000000000..d19c8080fd4d --- /dev/null +++ b/lib/test_min_heap.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define pr_fmt(fmt) "min_heap_test: " fmt + +/* + * Test cases for the min max heap. + */ + +#include <linux/log2.h> +#include <linux/min_heap.h> +#include <linux/module.h> +#include <linux/printk.h> +#include <linux/random.h> + +static __init bool less_than(const void *lhs, const void *rhs) +{ + return *(int *)lhs < *(int *)rhs; +} + +static __init bool greater_than(const void *lhs, const void *rhs) +{ + return *(int *)lhs > *(int *)rhs; +} + +static __init void swap_ints(void *lhs, void *rhs) +{ + int temp = *(int *)lhs; + + *(int *)lhs = *(int *)rhs; + *(int *)rhs = temp; +} + +static __init int pop_verify_heap(bool min_heap, + struct min_heap *heap, + const struct min_heap_callbacks *funcs) +{ + int *values = heap->data; + int err = 0; + int last; + + last = values[0]; + min_heap_pop(heap, funcs); + while (heap->nr > 0) { + if (min_heap) { + if (last > values[0]) { + pr_err("error: expected %d <= %d\n", last, + values[0]); + err++; + } + } else { + if (last < values[0]) { + pr_err("error: expected %d >= %d\n", last, + values[0]); + err++; + } + } + last = values[0]; + min_heap_pop(heap, funcs); + } + return err; +} + +static __init int test_heapify_all(bool min_heap) +{ + int values[] = { 3, 1, 2, 4, 0x8000000, 0x7FFFFFF, 0, + -3, -1, -2, -4, 0x8000000, 0x7FFFFFF }; + struct min_heap heap = { + .data = values, + .nr = ARRAY_SIZE(values), + .size = ARRAY_SIZE(values), + }; + struct min_heap_callbacks funcs = { + .elem_size = sizeof(int), + .less = min_heap ? less_than : greater_than, + .swp = swap_ints, + }; + int i, err; + + /* Test with known set of values. */ + min_heapify_all(&heap, &funcs); + err = pop_verify_heap(min_heap, &heap, &funcs); + + + /* Test with randomly generated values. */ + heap.nr = ARRAY_SIZE(values); + for (i = 0; i < heap.nr; i++) + values[i] = get_random_int(); + + min_heapify_all(&heap, &funcs); + err += pop_verify_heap(min_heap, &heap, &funcs); + + return err; +} + +static __init int test_heap_push(bool min_heap) +{ + const int data[] = { 3, 1, 2, 4, 0x80000000, 0x7FFFFFFF, 0, + -3, -1, -2, -4, 0x80000000, 0x7FFFFFFF }; + int values[ARRAY_SIZE(data)]; + struct min_heap heap = { + .data = values, + .nr = 0, + .size = ARRAY_SIZE(values), + }; + struct min_heap_callbacks funcs = { + .elem_size = sizeof(int), + .less = min_heap ? less_than : greater_than, + .swp = swap_ints, + }; + int i, temp, err; + + /* Test with known set of values copied from data. */ + for (i = 0; i < ARRAY_SIZE(data); i++) + min_heap_push(&heap, &data[i], &funcs); + + err = pop_verify_heap(min_heap, &heap, &funcs); + + /* Test with randomly generated values. */ + while (heap.nr < heap.size) { + temp = get_random_int(); + min_heap_push(&heap, &temp, &funcs); + } + err += pop_verify_heap(min_heap, &heap, &funcs); + + return err; +} + +static __init int test_heap_pop_push(bool min_heap) +{ + const int data[] = { 3, 1, 2, 4, 0x80000000, 0x7FFFFFFF, 0, + -3, -1, -2, -4, 0x80000000, 0x7FFFFFFF }; + int values[ARRAY_SIZE(data)]; + struct min_heap heap = { + .data = values, + .nr = 0, + .size = ARRAY_SIZE(values), + }; + struct min_heap_callbacks funcs = { + .elem_size = sizeof(int), + .less = min_heap ? less_than : greater_than, + .swp = swap_ints, + }; + int i, temp, err; + + /* Fill values with data to pop and replace. */ + temp = min_heap ? 0x80000000 : 0x7FFFFFFF; + for (i = 0; i < ARRAY_SIZE(data); i++) + min_heap_push(&heap, &temp, &funcs); + + /* Test with known set of values copied from data. */ + for (i = 0; i < ARRAY_SIZE(data); i++) + min_heap_pop_push(&heap, &data[i], &funcs); + + err = pop_verify_heap(min_heap, &heap, &funcs); + + heap.nr = 0; + for (i = 0; i < ARRAY_SIZE(data); i++) + min_heap_push(&heap, &temp, &funcs); + + /* Test with randomly generated values. */ + for (i = 0; i < ARRAY_SIZE(data); i++) { + temp = get_random_int(); + min_heap_pop_push(&heap, &temp, &funcs); + } + err += pop_verify_heap(min_heap, &heap, &funcs); + + return err; +} + +static int __init test_min_heap_init(void) +{ + int err = 0; + + err += test_heapify_all(true); + err += test_heapify_all(false); + err += test_heap_push(true); + err += test_heap_push(false); + err += test_heap_pop_push(true); + err += test_heap_pop_push(false); + if (err) { + pr_err("test failed with %d errors\n", err); + return -EINVAL; + } + pr_info("test passed\n"); + return 0; +} +module_init(test_min_heap_init); + +static void __exit test_min_heap_exit(void) +{ + /* do nothing */ +} +module_exit(test_min_heap_exit); + +MODULE_LICENSE("GPL"); diff --git a/lib/test_objagg.c b/lib/test_objagg.c index 72c1abfa154d..da137939a410 100644 --- a/lib/test_objagg.c +++ b/lib/test_objagg.c @@ -979,10 +979,10 @@ err_check_expect_stats2: err_world2_obj_get: for (i--; i >= 0; i--) world_obj_put(&world2, objagg, hints_case->key_ids[i]); - objagg_hints_put(hints); - objagg_destroy(objagg2); i = hints_case->key_ids_count; + objagg_destroy(objagg2); err_check_expect_hints_stats: + objagg_hints_put(hints); err_hints_get: err_check_expect_stats: err_world_obj_get: diff --git a/lib/test_printf.c b/lib/test_printf.c index 2d9f520d2f27..07309c45f327 100644 --- a/lib/test_printf.c +++ b/lib/test_printf.c @@ -30,11 +30,13 @@ #define PAD_SIZE 16 #define FILL_CHAR '$' -static unsigned total_tests __initdata; -static unsigned failed_tests __initdata; +KSTM_MODULE_GLOBALS(); + static char *test_buffer __initdata; static char *alloced_buffer __initdata; +extern bool no_hash_pointers; + static int __printf(4, 0) __init do_test(int bufsize, const char *expect, int elen, const char *fmt, va_list ap) @@ -214,6 +216,7 @@ test_string(void) #define PTR_STR "ffff0123456789ab" #define PTR_VAL_NO_CRNG "(____ptrval____)" #define ZEROS "00000000" /* hex 32 zero bits */ +#define ONES "ffffffff" /* hex 32 one bits */ static int __init plain_format(void) @@ -245,6 +248,7 @@ plain_format(void) #define PTR_STR "456789ab" #define PTR_VAL_NO_CRNG "(ptrval)" #define ZEROS "" +#define ONES "" static int __init plain_format(void) @@ -299,6 +303,12 @@ plain(void) { int err; + if (no_hash_pointers) { + pr_warn("skipping plain 'p' tests"); + skipped_tests += 2; + return; + } + err = plain_hash(); if (err) { pr_warn("plain 'p' does not appear to be hashed\n"); @@ -330,14 +340,28 @@ test_hashed(const char *fmt, const void *p) test(buf, fmt, p); } +/* + * NULL pointers aren't hashed. + */ static void __init null_pointer(void) { - test_hashed("%p", NULL); + test(ZEROS "00000000", "%p", NULL); test(ZEROS "00000000", "%px", NULL); test("(null)", "%pE", NULL); } +/* + * Error pointers aren't hashed. + */ +static void __init +error_pointer(void) +{ + test(ONES "fffffff5", "%p", ERR_PTR(-11)); + test(ONES "fffffff5", "%px", ERR_PTR(-11)); + test("(efault)", "%pE", ERR_PTR(-11)); +} + #define PTR_INVALID ((void *)0x000000ab) static void __init @@ -478,7 +502,7 @@ struct_va_format(void) } static void __init -struct_rtc_time(void) +time_and_date(void) { /* 1543210543 */ const struct rtc_time tm = { @@ -489,14 +513,26 @@ struct_rtc_time(void) .tm_mon = 10, .tm_year = 118, }; + /* 2019-01-04T15:32:23 */ + time64_t t = 1546615943; - test("(%ptR?)", "%pt", &tm); + test("(%pt?)", "%pt", &tm); test("2018-11-26T05:35:43", "%ptR", &tm); test("0118-10-26T05:35:43", "%ptRr", &tm); test("05:35:43|2018-11-26", "%ptRt|%ptRd", &tm, &tm); test("05:35:43|0118-10-26", "%ptRtr|%ptRdr", &tm, &tm); test("05:35:43|2018-11-26", "%ptRttr|%ptRdtr", &tm, &tm); test("05:35:43 tr|2018-11-26 tr", "%ptRt tr|%ptRd tr", &tm, &tm); + + test("2019-01-04T15:32:23", "%ptT", &t); + test("0119-00-04T15:32:23", "%ptTr", &t); + test("15:32:23|2019-01-04", "%ptTt|%ptTd", &t, &t); + test("15:32:23|0119-00-04", "%ptTtr|%ptTdr", &t, &t); + + test("2019-01-04 15:32:23", "%ptTs", &t); + test("0119-00-04 15:32:23", "%ptTsr", &t); + test("15:32:23|2019-01-04", "%ptTts|%ptTds", &t, &t); + test("15:32:23|0119-00-04", "%ptTtrs|%ptTdrs", &t, &t); } static void __init @@ -546,28 +582,90 @@ netdev_features(void) { } +struct page_flags_test { + int width; + int shift; + int mask; + const char *fmt; + const char *name; +}; + +static const struct page_flags_test pft[] = { + {SECTIONS_WIDTH, SECTIONS_PGSHIFT, SECTIONS_MASK, + "%d", "section"}, + {NODES_WIDTH, NODES_PGSHIFT, NODES_MASK, + "%d", "node"}, + {ZONES_WIDTH, ZONES_PGSHIFT, ZONES_MASK, + "%d", "zone"}, + {LAST_CPUPID_WIDTH, LAST_CPUPID_PGSHIFT, LAST_CPUPID_MASK, + "%#x", "lastcpupid"}, + {KASAN_TAG_WIDTH, KASAN_TAG_PGSHIFT, KASAN_TAG_MASK, + "%#x", "kasantag"}, +}; + +static void __init +page_flags_test(int section, int node, int zone, int last_cpupid, + int kasan_tag, unsigned long flags, const char *name, + char *cmp_buf) +{ + unsigned long values[] = {section, node, zone, last_cpupid, kasan_tag}; + unsigned long size; + bool append = false; + int i; + + for (i = 0; i < ARRAY_SIZE(values); i++) + flags |= (values[i] & pft[i].mask) << pft[i].shift; + + size = scnprintf(cmp_buf, BUF_SIZE, "%#lx(", flags); + if (flags & PAGEFLAGS_MASK) { + size += scnprintf(cmp_buf + size, BUF_SIZE - size, "%s", name); + append = true; + } + + for (i = 0; i < ARRAY_SIZE(pft); i++) { + if (!pft[i].width) + continue; + + if (append) + size += scnprintf(cmp_buf + size, BUF_SIZE - size, "|"); + + size += scnprintf(cmp_buf + size, BUF_SIZE - size, "%s=", + pft[i].name); + size += scnprintf(cmp_buf + size, BUF_SIZE - size, pft[i].fmt, + values[i] & pft[i].mask); + append = true; + } + + snprintf(cmp_buf + size, BUF_SIZE - size, ")"); + + test(cmp_buf, "%pGp", &flags); +} + static void __init flags(void) { unsigned long flags; - gfp_t gfp; char *cmp_buffer; + gfp_t gfp; + + cmp_buffer = kmalloc(BUF_SIZE, GFP_KERNEL); + if (!cmp_buffer) + return; flags = 0; - test("", "%pGp", &flags); + page_flags_test(0, 0, 0, 0, 0, flags, "", cmp_buffer); - /* Page flags should filter the zone id */ flags = 1UL << NR_PAGEFLAGS; - test("", "%pGp", &flags); + page_flags_test(0, 0, 0, 0, 0, flags, "", cmp_buffer); flags |= 1UL << PG_uptodate | 1UL << PG_dirty | 1UL << PG_lru | 1UL << PG_active | 1UL << PG_swapbacked; - test("uptodate|dirty|lru|active|swapbacked", "%pGp", &flags); - + page_flags_test(1, 1, 1, 0x1fffff, 1, flags, + "uptodate|dirty|lru|active|swapbacked", + cmp_buffer); - flags = VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC - | VM_DENYWRITE; - test("read|exec|mayread|maywrite|mayexec|denywrite", "%pGv", &flags); + flags = VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; + test("read|exec|mayread|maywrite|mayexec", "%pGv", &flags); gfp = GFP_TRANSHUGE; test("GFP_TRANSHUGE", "%pGg", &gfp); @@ -578,10 +676,6 @@ flags(void) gfp = __GFP_ATOMIC; test("__GFP_ATOMIC", "%pGg", &gfp); - cmp_buffer = kmalloc(BUF_SIZE, GFP_KERNEL); - if (!cmp_buffer) - return; - /* Any flags not translated by the table should remain numeric */ gfp = ~__GFP_BITS_MASK; snprintf(cmp_buffer, BUF_SIZE, "%#lx", (unsigned long) gfp); @@ -624,6 +718,23 @@ static void __init fwnode_pointer(void) software_node_unregister_nodes(softnodes); } +static void __init fourcc_pointer(void) +{ + struct { + u32 code; + char *str; + } const try[] = { + { 0x3231564e, "NV12 little-endian (0x3231564e)", }, + { 0xb231564e, "NV12 big-endian (0xb231564e)", }, + { 0x10111213, ".... little-endian (0x10111213)", }, + { 0x20303159, "Y10 little-endian (0x20303159)", }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(try); i++) + test(try[i].str, "%p4cc", &try[i].code); +} + static void __init errptr(void) { @@ -649,6 +760,7 @@ test_pointer(void) { plain(); null_pointer(); + error_pointer(); invalid_pointer(); symbol_ptr(); kernel_ptr(); @@ -661,13 +773,14 @@ test_pointer(void) uuid(); dentry(); struct_va_format(); - struct_rtc_time(); + time_and_date(); struct_clk(); bitmap(); netdev_features(); flags(); errptr(); fwnode_pointer(); + fourcc_pointer(); } static void __init selftest(void) diff --git a/lib/test_rhashtable.c b/lib/test_rhashtable.c index c5a6fef7b45d..5a1dd4736b56 100644 --- a/lib/test_rhashtable.c +++ b/lib/test_rhashtable.c @@ -434,7 +434,7 @@ static int __init test_rhltable(unsigned int entries) } else { if (WARN(err != -ENOENT, "removed non-existent element, error %d not %d", err, -ENOENT)) - continue; + continue; } } @@ -487,6 +487,7 @@ static unsigned int __init print_ht(struct rhltable *rhlt) struct rhashtable *ht; const struct bucket_table *tbl; char buff[512] = ""; + int offset = 0; unsigned int i, cnt = 0; ht = &rhlt->ht; @@ -501,18 +502,18 @@ static unsigned int __init print_ht(struct rhltable *rhlt) next = !rht_is_a_nulls(pos) ? rht_dereference(pos->next, ht) : NULL; if (!rht_is_a_nulls(pos)) { - sprintf(buff, "%s\nbucket[%d] -> ", buff, i); + offset += sprintf(buff + offset, "\nbucket[%d] -> ", i); } while (!rht_is_a_nulls(pos)) { struct rhlist_head *list = container_of(pos, struct rhlist_head, rhead); - sprintf(buff, "%s[[", buff); + offset += sprintf(buff + offset, "[["); do { pos = &list->rhead; list = rht_dereference(list->next, ht); p = rht_obj(ht, pos); - sprintf(buff, "%s val %d (tid=%d)%s", buff, p->value.id, p->value.tid, + offset += sprintf(buff + offset, " val %d (tid=%d)%s", p->value.id, p->value.tid, list? ", " : " "); cnt++; } while (list); @@ -521,7 +522,7 @@ static unsigned int __init print_ht(struct rhltable *rhlt) next = !rht_is_a_nulls(pos) ? rht_dereference(pos->next, ht) : NULL; - sprintf(buff, "%s]]%s", buff, !rht_is_a_nulls(pos) ? " -> " : ""); + offset += sprintf(buff + offset, "]]%s", !rht_is_a_nulls(pos) ? " -> " : ""); } } printk(KERN_ERR "\n---- ht: ----%s\n-------------\n", buff); diff --git a/lib/test_scanf.c b/lib/test_scanf.c new file mode 100644 index 000000000000..b620cf7de503 --- /dev/null +++ b/lib/test_scanf.c @@ -0,0 +1,813 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Test cases for sscanf facility. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/bitops.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/overflow.h> +#include <linux/printk.h> +#include <linux/random.h> +#include <linux/slab.h> +#include <linux/string.h> + +#include "../tools/testing/selftests/kselftest_module.h" + +#define BUF_SIZE 1024 + +KSTM_MODULE_GLOBALS(); +static char *test_buffer __initdata; +static char *fmt_buffer __initdata; +static struct rnd_state rnd_state __initdata; + +typedef int (*check_fn)(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap); + +static void __scanf(4, 6) __init +_test(check_fn fn, const void *check_data, const char *string, const char *fmt, + int n_args, ...) +{ + va_list ap, ap_copy; + int ret; + + total_tests++; + + va_start(ap, n_args); + va_copy(ap_copy, ap); + ret = vsscanf(string, fmt, ap_copy); + va_end(ap_copy); + + if (ret != n_args) { + pr_warn("vsscanf(\"%s\", \"%s\", ...) returned %d expected %d\n", + string, fmt, ret, n_args); + goto fail; + } + + ret = (*fn)(check_data, string, fmt, n_args, ap); + if (ret) + goto fail; + + va_end(ap); + + return; + +fail: + failed_tests++; + va_end(ap); +} + +#define _check_numbers_template(arg_fmt, expect, str, fmt, n_args, ap) \ +do { \ + pr_debug("\"%s\", \"%s\" ->\n", str, fmt); \ + for (; n_args > 0; n_args--, expect++) { \ + typeof(*expect) got = *va_arg(ap, typeof(expect)); \ + pr_debug("\t" arg_fmt "\n", got); \ + if (got != *expect) { \ + pr_warn("vsscanf(\"%s\", \"%s\", ...) expected " arg_fmt " got " arg_fmt "\n", \ + str, fmt, *expect, got); \ + return 1; \ + } \ + } \ + return 0; \ +} while (0) + +static int __init check_ull(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap) +{ + const unsigned long long *pval = check_data; + + _check_numbers_template("%llu", pval, string, fmt, n_args, ap); +} + +static int __init check_ll(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap) +{ + const long long *pval = check_data; + + _check_numbers_template("%lld", pval, string, fmt, n_args, ap); +} + +static int __init check_ulong(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap) +{ + const unsigned long *pval = check_data; + + _check_numbers_template("%lu", pval, string, fmt, n_args, ap); +} + +static int __init check_long(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap) +{ + const long *pval = check_data; + + _check_numbers_template("%ld", pval, string, fmt, n_args, ap); +} + +static int __init check_uint(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap) +{ + const unsigned int *pval = check_data; + + _check_numbers_template("%u", pval, string, fmt, n_args, ap); +} + +static int __init check_int(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap) +{ + const int *pval = check_data; + + _check_numbers_template("%d", pval, string, fmt, n_args, ap); +} + +static int __init check_ushort(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap) +{ + const unsigned short *pval = check_data; + + _check_numbers_template("%hu", pval, string, fmt, n_args, ap); +} + +static int __init check_short(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap) +{ + const short *pval = check_data; + + _check_numbers_template("%hd", pval, string, fmt, n_args, ap); +} + +static int __init check_uchar(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap) +{ + const unsigned char *pval = check_data; + + _check_numbers_template("%hhu", pval, string, fmt, n_args, ap); +} + +static int __init check_char(const void *check_data, const char *string, + const char *fmt, int n_args, va_list ap) +{ + const signed char *pval = check_data; + + _check_numbers_template("%hhd", pval, string, fmt, n_args, ap); +} + +/* Selection of interesting numbers to test, copied from test-kstrtox.c */ +static const unsigned long long numbers[] __initconst = { + 0x0ULL, + 0x1ULL, + 0x7fULL, + 0x80ULL, + 0x81ULL, + 0xffULL, + 0x100ULL, + 0x101ULL, + 0x7fffULL, + 0x8000ULL, + 0x8001ULL, + 0xffffULL, + 0x10000ULL, + 0x10001ULL, + 0x7fffffffULL, + 0x80000000ULL, + 0x80000001ULL, + 0xffffffffULL, + 0x100000000ULL, + 0x100000001ULL, + 0x7fffffffffffffffULL, + 0x8000000000000000ULL, + 0x8000000000000001ULL, + 0xfffffffffffffffeULL, + 0xffffffffffffffffULL, +}; + +#define value_representable_in_type(T, val) \ +(is_signed_type(T) \ + ? ((long long)(val) >= type_min(T)) && ((long long)(val) <= type_max(T)) \ + : ((unsigned long long)(val) <= type_max(T))) + + +#define test_one_number(T, gen_fmt, scan_fmt, val, fn) \ +do { \ + const T expect_val = (T)(val); \ + T result = ~expect_val; /* should be overwritten */ \ + \ + snprintf(test_buffer, BUF_SIZE, gen_fmt, expect_val); \ + _test(fn, &expect_val, test_buffer, "%" scan_fmt, 1, &result); \ +} while (0) + +#define simple_numbers_loop(T, gen_fmt, scan_fmt, fn) \ +do { \ + int i; \ + \ + for (i = 0; i < ARRAY_SIZE(numbers); i++) { \ + if (value_representable_in_type(T, numbers[i])) \ + test_one_number(T, gen_fmt, scan_fmt, \ + numbers[i], fn); \ + \ + if (value_representable_in_type(T, -numbers[i])) \ + test_one_number(T, gen_fmt, scan_fmt, \ + -numbers[i], fn); \ + } \ +} while (0) + +static void __init numbers_simple(void) +{ + simple_numbers_loop(unsigned long long, "%llu", "llu", check_ull); + simple_numbers_loop(long long, "%lld", "lld", check_ll); + simple_numbers_loop(long long, "%lld", "lli", check_ll); + simple_numbers_loop(unsigned long long, "%llx", "llx", check_ull); + simple_numbers_loop(long long, "%llx", "llx", check_ll); + simple_numbers_loop(long long, "0x%llx", "lli", check_ll); + simple_numbers_loop(unsigned long long, "0x%llx", "llx", check_ull); + simple_numbers_loop(long long, "0x%llx", "llx", check_ll); + + simple_numbers_loop(unsigned long, "%lu", "lu", check_ulong); + simple_numbers_loop(long, "%ld", "ld", check_long); + simple_numbers_loop(long, "%ld", "li", check_long); + simple_numbers_loop(unsigned long, "%lx", "lx", check_ulong); + simple_numbers_loop(long, "%lx", "lx", check_long); + simple_numbers_loop(long, "0x%lx", "li", check_long); + simple_numbers_loop(unsigned long, "0x%lx", "lx", check_ulong); + simple_numbers_loop(long, "0x%lx", "lx", check_long); + + simple_numbers_loop(unsigned int, "%u", "u", check_uint); + simple_numbers_loop(int, "%d", "d", check_int); + simple_numbers_loop(int, "%d", "i", check_int); + simple_numbers_loop(unsigned int, "%x", "x", check_uint); + simple_numbers_loop(int, "%x", "x", check_int); + simple_numbers_loop(int, "0x%x", "i", check_int); + simple_numbers_loop(unsigned int, "0x%x", "x", check_uint); + simple_numbers_loop(int, "0x%x", "x", check_int); + + simple_numbers_loop(unsigned short, "%hu", "hu", check_ushort); + simple_numbers_loop(short, "%hd", "hd", check_short); + simple_numbers_loop(short, "%hd", "hi", check_short); + simple_numbers_loop(unsigned short, "%hx", "hx", check_ushort); + simple_numbers_loop(short, "%hx", "hx", check_short); + simple_numbers_loop(short, "0x%hx", "hi", check_short); + simple_numbers_loop(unsigned short, "0x%hx", "hx", check_ushort); + simple_numbers_loop(short, "0x%hx", "hx", check_short); + + simple_numbers_loop(unsigned char, "%hhu", "hhu", check_uchar); + simple_numbers_loop(signed char, "%hhd", "hhd", check_char); + simple_numbers_loop(signed char, "%hhd", "hhi", check_char); + simple_numbers_loop(unsigned char, "%hhx", "hhx", check_uchar); + simple_numbers_loop(signed char, "%hhx", "hhx", check_char); + simple_numbers_loop(signed char, "0x%hhx", "hhi", check_char); + simple_numbers_loop(unsigned char, "0x%hhx", "hhx", check_uchar); + simple_numbers_loop(signed char, "0x%hhx", "hhx", check_char); +} + +/* + * This gives a better variety of number "lengths" in a small sample than + * the raw prandom*() functions (Not mathematically rigorous!!). + * Variabilty of length and value is more important than perfect randomness. + */ +static u32 __init next_test_random(u32 max_bits) +{ + u32 n_bits = hweight32(prandom_u32_state(&rnd_state)) % (max_bits + 1); + + return prandom_u32_state(&rnd_state) & GENMASK(n_bits, 0); +} + +static unsigned long long __init next_test_random_ull(void) +{ + u32 rand1 = prandom_u32_state(&rnd_state); + u32 n_bits = (hweight32(rand1) * 3) % 64; + u64 val = (u64)prandom_u32_state(&rnd_state) * rand1; + + return val & GENMASK_ULL(n_bits, 0); +} + +#define random_for_type(T) \ + ((T)(sizeof(T) <= sizeof(u32) \ + ? next_test_random(BITS_PER_TYPE(T)) \ + : next_test_random_ull())) + +/* + * Define a pattern of negative and positive numbers to ensure we get + * some of both within the small number of samples in a test string. + */ +#define NEGATIVES_PATTERN 0x3246 /* 00110010 01000110 */ + +#define fill_random_array(arr) \ +do { \ + unsigned int neg_pattern = NEGATIVES_PATTERN; \ + int i; \ + \ + for (i = 0; i < ARRAY_SIZE(arr); i++, neg_pattern >>= 1) { \ + (arr)[i] = random_for_type(typeof((arr)[0])); \ + if (is_signed_type(typeof((arr)[0])) && (neg_pattern & 1)) \ + (arr)[i] = -(arr)[i]; \ + } \ +} while (0) + +/* + * Convenience wrapper around snprintf() to append at buf_pos in buf, + * updating buf_pos and returning the number of characters appended. + * On error buf_pos is not changed and return value is 0. + */ +static int __init __printf(4, 5) +append_fmt(char *buf, int *buf_pos, int buf_len, const char *val_fmt, ...) +{ + va_list ap; + int field_len; + + va_start(ap, val_fmt); + field_len = vsnprintf(buf + *buf_pos, buf_len - *buf_pos, val_fmt, ap); + va_end(ap); + + if (field_len < 0) + field_len = 0; + + *buf_pos += field_len; + + return field_len; +} + +/* + * Convenience function to append the field delimiter string + * to both the value string and format string buffers. + */ +static void __init append_delim(char *str_buf, int *str_buf_pos, int str_buf_len, + char *fmt_buf, int *fmt_buf_pos, int fmt_buf_len, + const char *delim_str) +{ + append_fmt(str_buf, str_buf_pos, str_buf_len, delim_str); + append_fmt(fmt_buf, fmt_buf_pos, fmt_buf_len, delim_str); +} + +#define test_array_8(fn, check_data, string, fmt, arr) \ +do { \ + BUILD_BUG_ON(ARRAY_SIZE(arr) != 8); \ + _test(fn, check_data, string, fmt, 8, \ + &(arr)[0], &(arr)[1], &(arr)[2], &(arr)[3], \ + &(arr)[4], &(arr)[5], &(arr)[6], &(arr)[7]); \ +} while (0) + +#define numbers_list_8(T, gen_fmt, field_sep, scan_fmt, fn) \ +do { \ + int i, pos = 0, fmt_pos = 0; \ + T expect[8], result[8]; \ + \ + fill_random_array(expect); \ + \ + for (i = 0; i < ARRAY_SIZE(expect); i++) { \ + if (i != 0) \ + append_delim(test_buffer, &pos, BUF_SIZE, \ + fmt_buffer, &fmt_pos, BUF_SIZE, \ + field_sep); \ + \ + append_fmt(test_buffer, &pos, BUF_SIZE, gen_fmt, expect[i]); \ + append_fmt(fmt_buffer, &fmt_pos, BUF_SIZE, "%%%s", scan_fmt); \ + } \ + \ + test_array_8(fn, expect, test_buffer, fmt_buffer, result); \ +} while (0) + +#define numbers_list_fix_width(T, gen_fmt, field_sep, width, scan_fmt, fn) \ +do { \ + char full_fmt[16]; \ + \ + snprintf(full_fmt, sizeof(full_fmt), "%u%s", width, scan_fmt); \ + numbers_list_8(T, gen_fmt, field_sep, full_fmt, fn); \ +} while (0) + +#define numbers_list_val_width(T, gen_fmt, field_sep, scan_fmt, fn) \ +do { \ + int i, val_len, pos = 0, fmt_pos = 0; \ + T expect[8], result[8]; \ + \ + fill_random_array(expect); \ + \ + for (i = 0; i < ARRAY_SIZE(expect); i++) { \ + if (i != 0) \ + append_delim(test_buffer, &pos, BUF_SIZE, \ + fmt_buffer, &fmt_pos, BUF_SIZE, field_sep);\ + \ + val_len = append_fmt(test_buffer, &pos, BUF_SIZE, gen_fmt, \ + expect[i]); \ + append_fmt(fmt_buffer, &fmt_pos, BUF_SIZE, \ + "%%%u%s", val_len, scan_fmt); \ + } \ + \ + test_array_8(fn, expect, test_buffer, fmt_buffer, result); \ +} while (0) + +static void __init numbers_list_ll(const char *delim) +{ + numbers_list_8(unsigned long long, "%llu", delim, "llu", check_ull); + numbers_list_8(long long, "%lld", delim, "lld", check_ll); + numbers_list_8(long long, "%lld", delim, "lli", check_ll); + numbers_list_8(unsigned long long, "%llx", delim, "llx", check_ull); + numbers_list_8(unsigned long long, "0x%llx", delim, "llx", check_ull); + numbers_list_8(long long, "0x%llx", delim, "lli", check_ll); +} + +static void __init numbers_list_l(const char *delim) +{ + numbers_list_8(unsigned long, "%lu", delim, "lu", check_ulong); + numbers_list_8(long, "%ld", delim, "ld", check_long); + numbers_list_8(long, "%ld", delim, "li", check_long); + numbers_list_8(unsigned long, "%lx", delim, "lx", check_ulong); + numbers_list_8(unsigned long, "0x%lx", delim, "lx", check_ulong); + numbers_list_8(long, "0x%lx", delim, "li", check_long); +} + +static void __init numbers_list_d(const char *delim) +{ + numbers_list_8(unsigned int, "%u", delim, "u", check_uint); + numbers_list_8(int, "%d", delim, "d", check_int); + numbers_list_8(int, "%d", delim, "i", check_int); + numbers_list_8(unsigned int, "%x", delim, "x", check_uint); + numbers_list_8(unsigned int, "0x%x", delim, "x", check_uint); + numbers_list_8(int, "0x%x", delim, "i", check_int); +} + +static void __init numbers_list_h(const char *delim) +{ + numbers_list_8(unsigned short, "%hu", delim, "hu", check_ushort); + numbers_list_8(short, "%hd", delim, "hd", check_short); + numbers_list_8(short, "%hd", delim, "hi", check_short); + numbers_list_8(unsigned short, "%hx", delim, "hx", check_ushort); + numbers_list_8(unsigned short, "0x%hx", delim, "hx", check_ushort); + numbers_list_8(short, "0x%hx", delim, "hi", check_short); +} + +static void __init numbers_list_hh(const char *delim) +{ + numbers_list_8(unsigned char, "%hhu", delim, "hhu", check_uchar); + numbers_list_8(signed char, "%hhd", delim, "hhd", check_char); + numbers_list_8(signed char, "%hhd", delim, "hhi", check_char); + numbers_list_8(unsigned char, "%hhx", delim, "hhx", check_uchar); + numbers_list_8(unsigned char, "0x%hhx", delim, "hhx", check_uchar); + numbers_list_8(signed char, "0x%hhx", delim, "hhi", check_char); +} + +static void __init numbers_list(const char *delim) +{ + numbers_list_ll(delim); + numbers_list_l(delim); + numbers_list_d(delim); + numbers_list_h(delim); + numbers_list_hh(delim); +} + +static void __init numbers_list_field_width_ll(const char *delim) +{ + numbers_list_fix_width(unsigned long long, "%llu", delim, 20, "llu", check_ull); + numbers_list_fix_width(long long, "%lld", delim, 20, "lld", check_ll); + numbers_list_fix_width(long long, "%lld", delim, 20, "lli", check_ll); + numbers_list_fix_width(unsigned long long, "%llx", delim, 16, "llx", check_ull); + numbers_list_fix_width(unsigned long long, "0x%llx", delim, 18, "llx", check_ull); + numbers_list_fix_width(long long, "0x%llx", delim, 18, "lli", check_ll); +} + +static void __init numbers_list_field_width_l(const char *delim) +{ +#if BITS_PER_LONG == 64 + numbers_list_fix_width(unsigned long, "%lu", delim, 20, "lu", check_ulong); + numbers_list_fix_width(long, "%ld", delim, 20, "ld", check_long); + numbers_list_fix_width(long, "%ld", delim, 20, "li", check_long); + numbers_list_fix_width(unsigned long, "%lx", delim, 16, "lx", check_ulong); + numbers_list_fix_width(unsigned long, "0x%lx", delim, 18, "lx", check_ulong); + numbers_list_fix_width(long, "0x%lx", delim, 18, "li", check_long); +#else + numbers_list_fix_width(unsigned long, "%lu", delim, 10, "lu", check_ulong); + numbers_list_fix_width(long, "%ld", delim, 11, "ld", check_long); + numbers_list_fix_width(long, "%ld", delim, 11, "li", check_long); + numbers_list_fix_width(unsigned long, "%lx", delim, 8, "lx", check_ulong); + numbers_list_fix_width(unsigned long, "0x%lx", delim, 10, "lx", check_ulong); + numbers_list_fix_width(long, "0x%lx", delim, 10, "li", check_long); +#endif +} + +static void __init numbers_list_field_width_d(const char *delim) +{ + numbers_list_fix_width(unsigned int, "%u", delim, 10, "u", check_uint); + numbers_list_fix_width(int, "%d", delim, 11, "d", check_int); + numbers_list_fix_width(int, "%d", delim, 11, "i", check_int); + numbers_list_fix_width(unsigned int, "%x", delim, 8, "x", check_uint); + numbers_list_fix_width(unsigned int, "0x%x", delim, 10, "x", check_uint); + numbers_list_fix_width(int, "0x%x", delim, 10, "i", check_int); +} + +static void __init numbers_list_field_width_h(const char *delim) +{ + numbers_list_fix_width(unsigned short, "%hu", delim, 5, "hu", check_ushort); + numbers_list_fix_width(short, "%hd", delim, 6, "hd", check_short); + numbers_list_fix_width(short, "%hd", delim, 6, "hi", check_short); + numbers_list_fix_width(unsigned short, "%hx", delim, 4, "hx", check_ushort); + numbers_list_fix_width(unsigned short, "0x%hx", delim, 6, "hx", check_ushort); + numbers_list_fix_width(short, "0x%hx", delim, 6, "hi", check_short); +} + +static void __init numbers_list_field_width_hh(const char *delim) +{ + numbers_list_fix_width(unsigned char, "%hhu", delim, 3, "hhu", check_uchar); + numbers_list_fix_width(signed char, "%hhd", delim, 4, "hhd", check_char); + numbers_list_fix_width(signed char, "%hhd", delim, 4, "hhi", check_char); + numbers_list_fix_width(unsigned char, "%hhx", delim, 2, "hhx", check_uchar); + numbers_list_fix_width(unsigned char, "0x%hhx", delim, 4, "hhx", check_uchar); + numbers_list_fix_width(signed char, "0x%hhx", delim, 4, "hhi", check_char); +} + +/* + * List of numbers separated by delim. Each field width specifier is the + * maximum possible digits for the given type and base. + */ +static void __init numbers_list_field_width_typemax(const char *delim) +{ + numbers_list_field_width_ll(delim); + numbers_list_field_width_l(delim); + numbers_list_field_width_d(delim); + numbers_list_field_width_h(delim); + numbers_list_field_width_hh(delim); +} + +static void __init numbers_list_field_width_val_ll(const char *delim) +{ + numbers_list_val_width(unsigned long long, "%llu", delim, "llu", check_ull); + numbers_list_val_width(long long, "%lld", delim, "lld", check_ll); + numbers_list_val_width(long long, "%lld", delim, "lli", check_ll); + numbers_list_val_width(unsigned long long, "%llx", delim, "llx", check_ull); + numbers_list_val_width(unsigned long long, "0x%llx", delim, "llx", check_ull); + numbers_list_val_width(long long, "0x%llx", delim, "lli", check_ll); +} + +static void __init numbers_list_field_width_val_l(const char *delim) +{ + numbers_list_val_width(unsigned long, "%lu", delim, "lu", check_ulong); + numbers_list_val_width(long, "%ld", delim, "ld", check_long); + numbers_list_val_width(long, "%ld", delim, "li", check_long); + numbers_list_val_width(unsigned long, "%lx", delim, "lx", check_ulong); + numbers_list_val_width(unsigned long, "0x%lx", delim, "lx", check_ulong); + numbers_list_val_width(long, "0x%lx", delim, "li", check_long); +} + +static void __init numbers_list_field_width_val_d(const char *delim) +{ + numbers_list_val_width(unsigned int, "%u", delim, "u", check_uint); + numbers_list_val_width(int, "%d", delim, "d", check_int); + numbers_list_val_width(int, "%d", delim, "i", check_int); + numbers_list_val_width(unsigned int, "%x", delim, "x", check_uint); + numbers_list_val_width(unsigned int, "0x%x", delim, "x", check_uint); + numbers_list_val_width(int, "0x%x", delim, "i", check_int); +} + +static void __init numbers_list_field_width_val_h(const char *delim) +{ + numbers_list_val_width(unsigned short, "%hu", delim, "hu", check_ushort); + numbers_list_val_width(short, "%hd", delim, "hd", check_short); + numbers_list_val_width(short, "%hd", delim, "hi", check_short); + numbers_list_val_width(unsigned short, "%hx", delim, "hx", check_ushort); + numbers_list_val_width(unsigned short, "0x%hx", delim, "hx", check_ushort); + numbers_list_val_width(short, "0x%hx", delim, "hi", check_short); +} + +static void __init numbers_list_field_width_val_hh(const char *delim) +{ + numbers_list_val_width(unsigned char, "%hhu", delim, "hhu", check_uchar); + numbers_list_val_width(signed char, "%hhd", delim, "hhd", check_char); + numbers_list_val_width(signed char, "%hhd", delim, "hhi", check_char); + numbers_list_val_width(unsigned char, "%hhx", delim, "hhx", check_uchar); + numbers_list_val_width(unsigned char, "0x%hhx", delim, "hhx", check_uchar); + numbers_list_val_width(signed char, "0x%hhx", delim, "hhi", check_char); +} + +/* + * List of numbers separated by delim. Each field width specifier is the + * exact length of the corresponding value digits in the string being scanned. + */ +static void __init numbers_list_field_width_val_width(const char *delim) +{ + numbers_list_field_width_val_ll(delim); + numbers_list_field_width_val_l(delim); + numbers_list_field_width_val_d(delim); + numbers_list_field_width_val_h(delim); + numbers_list_field_width_val_hh(delim); +} + +/* + * Slice a continuous string of digits without field delimiters, containing + * numbers of varying length, using the field width to extract each group + * of digits. For example the hex values c0,3,bf01,303 would have a + * string representation of "c03bf01303" and extracted with "%2x%1x%4x%3x". + */ +static void __init numbers_slice(void) +{ + numbers_list_field_width_val_width(""); +} + +#define test_number_prefix(T, str, scan_fmt, expect0, expect1, n_args, fn) \ +do { \ + const T expect[2] = { expect0, expect1 }; \ + T result[2] = {~expect[0], ~expect[1]}; \ + \ + _test(fn, &expect, str, scan_fmt, n_args, &result[0], &result[1]); \ +} while (0) + +/* + * Number prefix is >= field width. + * Expected behaviour is derived from testing userland sscanf. + */ +static void __init numbers_prefix_overflow(void) +{ + /* + * Negative decimal with a field of width 1, should quit scanning + * and return 0. + */ + test_number_prefix(long long, "-1 1", "%1lld %lld", 0, 0, 0, check_ll); + test_number_prefix(long, "-1 1", "%1ld %ld", 0, 0, 0, check_long); + test_number_prefix(int, "-1 1", "%1d %d", 0, 0, 0, check_int); + test_number_prefix(short, "-1 1", "%1hd %hd", 0, 0, 0, check_short); + test_number_prefix(signed char, "-1 1", "%1hhd %hhd", 0, 0, 0, check_char); + + test_number_prefix(long long, "-1 1", "%1lli %lli", 0, 0, 0, check_ll); + test_number_prefix(long, "-1 1", "%1li %li", 0, 0, 0, check_long); + test_number_prefix(int, "-1 1", "%1i %i", 0, 0, 0, check_int); + test_number_prefix(short, "-1 1", "%1hi %hi", 0, 0, 0, check_short); + test_number_prefix(signed char, "-1 1", "%1hhi %hhi", 0, 0, 0, check_char); + + /* + * 0x prefix in a field of width 1: 0 is a valid digit so should + * convert. Next field scan starts at the 'x' which isn't a digit so + * scan quits with one field converted. + */ + test_number_prefix(unsigned long long, "0xA7", "%1llx%llx", 0, 0, 1, check_ull); + test_number_prefix(unsigned long, "0xA7", "%1lx%lx", 0, 0, 1, check_ulong); + test_number_prefix(unsigned int, "0xA7", "%1x%x", 0, 0, 1, check_uint); + test_number_prefix(unsigned short, "0xA7", "%1hx%hx", 0, 0, 1, check_ushort); + test_number_prefix(unsigned char, "0xA7", "%1hhx%hhx", 0, 0, 1, check_uchar); + test_number_prefix(long long, "0xA7", "%1lli%llx", 0, 0, 1, check_ll); + test_number_prefix(long, "0xA7", "%1li%lx", 0, 0, 1, check_long); + test_number_prefix(int, "0xA7", "%1i%x", 0, 0, 1, check_int); + test_number_prefix(short, "0xA7", "%1hi%hx", 0, 0, 1, check_short); + test_number_prefix(char, "0xA7", "%1hhi%hhx", 0, 0, 1, check_char); + + /* + * 0x prefix in a field of width 2 using %x conversion: first field + * converts to 0. Next field scan starts at the character after "0x". + * Both fields will convert. + */ + test_number_prefix(unsigned long long, "0xA7", "%2llx%llx", 0, 0xa7, 2, check_ull); + test_number_prefix(unsigned long, "0xA7", "%2lx%lx", 0, 0xa7, 2, check_ulong); + test_number_prefix(unsigned int, "0xA7", "%2x%x", 0, 0xa7, 2, check_uint); + test_number_prefix(unsigned short, "0xA7", "%2hx%hx", 0, 0xa7, 2, check_ushort); + test_number_prefix(unsigned char, "0xA7", "%2hhx%hhx", 0, 0xa7, 2, check_uchar); + + /* + * 0x prefix in a field of width 2 using %i conversion: first field + * converts to 0. Next field scan starts at the character after "0x", + * which will convert if can be interpreted as decimal but will fail + * if it contains any hex digits (since no 0x prefix). + */ + test_number_prefix(long long, "0x67", "%2lli%lli", 0, 67, 2, check_ll); + test_number_prefix(long, "0x67", "%2li%li", 0, 67, 2, check_long); + test_number_prefix(int, "0x67", "%2i%i", 0, 67, 2, check_int); + test_number_prefix(short, "0x67", "%2hi%hi", 0, 67, 2, check_short); + test_number_prefix(char, "0x67", "%2hhi%hhi", 0, 67, 2, check_char); + + test_number_prefix(long long, "0xA7", "%2lli%lli", 0, 0, 1, check_ll); + test_number_prefix(long, "0xA7", "%2li%li", 0, 0, 1, check_long); + test_number_prefix(int, "0xA7", "%2i%i", 0, 0, 1, check_int); + test_number_prefix(short, "0xA7", "%2hi%hi", 0, 0, 1, check_short); + test_number_prefix(char, "0xA7", "%2hhi%hhi", 0, 0, 1, check_char); +} + +#define _test_simple_strtoxx(T, fn, gen_fmt, expect, base) \ +do { \ + T got; \ + char *endp; \ + int len; \ + bool fail = false; \ + \ + total_tests++; \ + len = snprintf(test_buffer, BUF_SIZE, gen_fmt, expect); \ + got = (fn)(test_buffer, &endp, base); \ + pr_debug(#fn "(\"%s\", %d) -> " gen_fmt "\n", test_buffer, base, got); \ + if (got != (expect)) { \ + fail = true; \ + pr_warn(#fn "(\"%s\", %d): got " gen_fmt " expected " gen_fmt "\n", \ + test_buffer, base, got, expect); \ + } else if (endp != test_buffer + len) { \ + fail = true; \ + pr_warn(#fn "(\"%s\", %d) startp=0x%px got endp=0x%px expected 0x%px\n", \ + test_buffer, base, test_buffer, \ + test_buffer + len, endp); \ + } \ + \ + if (fail) \ + failed_tests++; \ +} while (0) + +#define test_simple_strtoxx(T, fn, gen_fmt, base) \ +do { \ + int i; \ + \ + for (i = 0; i < ARRAY_SIZE(numbers); i++) { \ + _test_simple_strtoxx(T, fn, gen_fmt, (T)numbers[i], base); \ + \ + if (is_signed_type(T)) \ + _test_simple_strtoxx(T, fn, gen_fmt, \ + -(T)numbers[i], base); \ + } \ +} while (0) + +static void __init test_simple_strtoull(void) +{ + test_simple_strtoxx(unsigned long long, simple_strtoull, "%llu", 10); + test_simple_strtoxx(unsigned long long, simple_strtoull, "%llu", 0); + test_simple_strtoxx(unsigned long long, simple_strtoull, "%llx", 16); + test_simple_strtoxx(unsigned long long, simple_strtoull, "0x%llx", 16); + test_simple_strtoxx(unsigned long long, simple_strtoull, "0x%llx", 0); +} + +static void __init test_simple_strtoll(void) +{ + test_simple_strtoxx(long long, simple_strtoll, "%lld", 10); + test_simple_strtoxx(long long, simple_strtoll, "%lld", 0); + test_simple_strtoxx(long long, simple_strtoll, "%llx", 16); + test_simple_strtoxx(long long, simple_strtoll, "0x%llx", 16); + test_simple_strtoxx(long long, simple_strtoll, "0x%llx", 0); +} + +static void __init test_simple_strtoul(void) +{ + test_simple_strtoxx(unsigned long, simple_strtoul, "%lu", 10); + test_simple_strtoxx(unsigned long, simple_strtoul, "%lu", 0); + test_simple_strtoxx(unsigned long, simple_strtoul, "%lx", 16); + test_simple_strtoxx(unsigned long, simple_strtoul, "0x%lx", 16); + test_simple_strtoxx(unsigned long, simple_strtoul, "0x%lx", 0); +} + +static void __init test_simple_strtol(void) +{ + test_simple_strtoxx(long, simple_strtol, "%ld", 10); + test_simple_strtoxx(long, simple_strtol, "%ld", 0); + test_simple_strtoxx(long, simple_strtol, "%lx", 16); + test_simple_strtoxx(long, simple_strtol, "0x%lx", 16); + test_simple_strtoxx(long, simple_strtol, "0x%lx", 0); +} + +/* Selection of common delimiters/separators between numbers in a string. */ +static const char * const number_delimiters[] __initconst = { + " ", ":", ",", "-", "/", +}; + +static void __init test_numbers(void) +{ + int i; + + /* String containing only one number. */ + numbers_simple(); + + /* String with multiple numbers separated by delimiter. */ + for (i = 0; i < ARRAY_SIZE(number_delimiters); i++) { + numbers_list(number_delimiters[i]); + + /* Field width may be longer than actual field digits. */ + numbers_list_field_width_typemax(number_delimiters[i]); + + /* Each field width exactly length of actual field digits. */ + numbers_list_field_width_val_width(number_delimiters[i]); + } + + /* Slice continuous sequence of digits using field widths. */ + numbers_slice(); + + numbers_prefix_overflow(); +} + +static void __init selftest(void) +{ + test_buffer = kmalloc(BUF_SIZE, GFP_KERNEL); + if (!test_buffer) + return; + + fmt_buffer = kmalloc(BUF_SIZE, GFP_KERNEL); + if (!fmt_buffer) { + kfree(test_buffer); + return; + } + + prandom_seed_state(&rnd_state, 3141592653589793238ULL); + + test_numbers(); + + test_simple_strtoull(); + test_simple_strtoll(); + test_simple_strtoul(); + test_simple_strtol(); + + kfree(fmt_buffer); + kfree(test_buffer); +} + +KSTM_MODULE_LOADERS(test_scanf); +MODULE_AUTHOR("Richard Fitzgerald <rf@opensource.cirrus.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/lib/test_sort.c b/lib/test_sort.c index 52edbe10f2e5..be02e3a098cf 100644 --- a/lib/test_sort.c +++ b/lib/test_sort.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only + +#include <kunit/test.h> + #include <linux/sort.h> #include <linux/slab.h> #include <linux/module.h> @@ -7,18 +10,17 @@ #define TEST_LEN 1000 -static int __init cmpint(const void *a, const void *b) +static int cmpint(const void *a, const void *b) { return *(int *)a - *(int *)b; } -static int __init test_sort_init(void) +static void test_sort(struct kunit *test) { - int *a, i, r = 1, err = -ENOMEM; + int *a, i, r = 1; - a = kmalloc_array(TEST_LEN, sizeof(*a), GFP_KERNEL); - if (!a) - return err; + a = kunit_kmalloc_array(test, TEST_LEN, sizeof(*a), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, a); for (i = 0; i < TEST_LEN; i++) { r = (r * 725861) % 6599; @@ -27,24 +29,20 @@ static int __init test_sort_init(void) sort(a, TEST_LEN, sizeof(*a), cmpint, NULL); - err = -EINVAL; for (i = 0; i < TEST_LEN-1; i++) - if (a[i] > a[i+1]) { - pr_err("test has failed\n"); - goto exit; - } - err = 0; - pr_info("test passed\n"); -exit: - kfree(a); - return err; + KUNIT_ASSERT_LE(test, a[i], a[i + 1]); } -static void __exit test_sort_exit(void) -{ -} +static struct kunit_case sort_test_cases[] = { + KUNIT_CASE(test_sort), + {} +}; + +static struct kunit_suite sort_test_suite = { + .name = "lib_sort", + .test_cases = sort_test_cases, +}; -module_init(test_sort_init); -module_exit(test_sort_exit); +kunit_test_suites(&sort_test_suite); MODULE_LICENSE("GPL"); diff --git a/lib/test_stackinit.c b/lib/test_stackinit.c index 2d7d257a430e..a3c74e6a21ff 100644 --- a/lib/test_stackinit.c +++ b/lib/test_stackinit.c @@ -1,8 +1,13 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-or-later /* - * Test cases for compiler-based stack variable zeroing via future - * compiler flags or CONFIG_GCC_PLUGIN_STRUCTLEAK*. + * Test cases for compiler-based stack variable zeroing via + * -ftrivial-auto-var-init={zero,pattern} or CONFIG_GCC_PLUGIN_STRUCTLEAK*. + * + * External build example: + * clang -O2 -Wall -ftrivial-auto-var-init=pattern \ + * -o test_stackinit test_stackinit.c */ +#ifdef __KERNEL__ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/init.h> @@ -10,6 +15,63 @@ #include <linux/module.h> #include <linux/string.h> +#else + +/* Userspace headers. */ +#include <stdio.h> +#include <stdint.h> +#include <string.h> +#include <stdbool.h> +#include <errno.h> +#include <sys/types.h> + +/* Linux kernel-ism stubs for stand-alone userspace build. */ +#define KBUILD_MODNAME "stackinit" +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define pr_err(fmt, ...) fprintf(stderr, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_warn(fmt, ...) fprintf(stderr, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_info(fmt, ...) fprintf(stdout, pr_fmt(fmt), ##__VA_ARGS__) +#define __init /**/ +#define __exit /**/ +#define __user /**/ +#define noinline __attribute__((__noinline__)) +#define __aligned(x) __attribute__((__aligned__(x))) +#ifdef __clang__ +# define __compiletime_error(message) /**/ +#else +# define __compiletime_error(message) __attribute__((__error__(message))) +#endif +#define __compiletime_assert(condition, msg, prefix, suffix) \ + do { \ + extern void prefix ## suffix(void) __compiletime_error(msg); \ + if (!(condition)) \ + prefix ## suffix(); \ + } while (0) +#define _compiletime_assert(condition, msg, prefix, suffix) \ + __compiletime_assert(condition, msg, prefix, suffix) +#define compiletime_assert(condition, msg) \ + _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__) +#define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg) +#define BUILD_BUG_ON(condition) \ + BUILD_BUG_ON_MSG(condition, "BUILD_BUG_ON failed: " #condition) +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; + +#define module_init(func) static int (*do_init)(void) = func +#define module_exit(func) static void (*do_exit)(void) = func +#define MODULE_LICENSE(str) int main(void) { \ + int rc; \ + /* License: str */ \ + rc = do_init(); \ + if (rc == 0) \ + do_exit(); \ + return rc; \ + } + +#endif /* __KERNEL__ */ + /* Exfiltration buffer. */ #define MAX_VAR_SIZE 128 static u8 check_buf[MAX_VAR_SIZE]; @@ -33,6 +95,10 @@ static bool range_contains(char *haystack_start, size_t haystack_size, return false; } +/* Whether the test is expected to fail. */ +#define WANT_SUCCESS 0 +#define XFAIL 1 + #define DO_NOTHING_TYPE_SCALAR(var_type) var_type #define DO_NOTHING_TYPE_STRING(var_type) void #define DO_NOTHING_TYPE_STRUCT(var_type) void @@ -58,42 +124,82 @@ static bool range_contains(char *haystack_start, size_t haystack_size, #define INIT_CLONE_STRING [FILL_SIZE_STRING] #define INIT_CLONE_STRUCT /**/ -#define INIT_SCALAR_none /**/ -#define INIT_SCALAR_zero = 0 +#define ZERO_CLONE_SCALAR(zero) memset(&(zero), 0x00, sizeof(zero)) +#define ZERO_CLONE_STRING(zero) memset(&(zero), 0x00, sizeof(zero)) +/* + * For the struct, intentionally poison padding to see if it gets + * copied out in direct assignments. + * */ +#define ZERO_CLONE_STRUCT(zero) \ + do { \ + memset(&(zero), 0xFF, sizeof(zero)); \ + zero.one = 0; \ + zero.two = 0; \ + zero.three = 0; \ + zero.four = 0; \ + } while (0) + +#define INIT_SCALAR_none(var_type) /**/ +#define INIT_SCALAR_zero(var_type) = 0 + +#define INIT_STRING_none(var_type) [FILL_SIZE_STRING] /**/ +#define INIT_STRING_zero(var_type) [FILL_SIZE_STRING] = { } + +#define INIT_STRUCT_none(var_type) /**/ +#define INIT_STRUCT_zero(var_type) = { } -#define INIT_STRING_none [FILL_SIZE_STRING] /**/ -#define INIT_STRING_zero [FILL_SIZE_STRING] = { } -#define INIT_STRUCT_none /**/ -#define INIT_STRUCT_zero = { } -#define INIT_STRUCT_static_partial = { .two = 0, } -#define INIT_STRUCT_static_all = { .one = arg->one, \ - .two = arg->two, \ - .three = arg->three, \ - .four = arg->four, \ +#define __static_partial { .two = 0, } +#define __static_all { .one = 0, \ + .two = 0, \ + .three = 0, \ + .four = 0, \ } -#define INIT_STRUCT_dynamic_partial = { .two = arg->two, } -#define INIT_STRUCT_dynamic_all = { .one = arg->one, \ - .two = arg->two, \ - .three = arg->three, \ - .four = arg->four, \ +#define __dynamic_partial { .two = arg->two, } +#define __dynamic_all { .one = arg->one, \ + .two = arg->two, \ + .three = arg->three, \ + .four = arg->four, \ } -#define INIT_STRUCT_runtime_partial ; \ - var.two = 0 -#define INIT_STRUCT_runtime_all ; \ - var.one = 0; \ +#define __runtime_partial var.two = 0 +#define __runtime_all var.one = 0; \ var.two = 0; \ var.three = 0; \ - memset(&var.four, 0, \ - sizeof(var.four)) + var.four = 0 + +#define INIT_STRUCT_static_partial(var_type) \ + = __static_partial +#define INIT_STRUCT_static_all(var_type) \ + = __static_all +#define INIT_STRUCT_dynamic_partial(var_type) \ + = __dynamic_partial +#define INIT_STRUCT_dynamic_all(var_type) \ + = __dynamic_all +#define INIT_STRUCT_runtime_partial(var_type) \ + ; __runtime_partial +#define INIT_STRUCT_runtime_all(var_type) \ + ; __runtime_all + +#define INIT_STRUCT_assigned_static_partial(var_type) \ + ; var = (var_type)__static_partial +#define INIT_STRUCT_assigned_static_all(var_type) \ + ; var = (var_type)__static_all +#define INIT_STRUCT_assigned_dynamic_partial(var_type) \ + ; var = (var_type)__dynamic_partial +#define INIT_STRUCT_assigned_dynamic_all(var_type) \ + ; var = (var_type)__dynamic_all + +#define INIT_STRUCT_assigned_copy(var_type) \ + ; var = *(arg) /* * @name: unique string name for the test * @var_type: type to be tested for zeroing initialization * @which: is this a SCALAR, STRING, or STRUCT type? * @init_level: what kind of initialization is performed + * @xfail: is this test expected to fail? */ -#define DEFINE_TEST_DRIVER(name, var_type, which) \ +#define DEFINE_TEST_DRIVER(name, var_type, which, xfail) \ /* Returns 0 on success, 1 on failure. */ \ static noinline __init int test_ ## name (void) \ { \ @@ -105,7 +211,7 @@ static noinline __init int test_ ## name (void) \ BUILD_BUG_ON(sizeof(zero) > MAX_VAR_SIZE); \ \ /* Fill clone type with zero for per-field init. */ \ - memset(&zero, 0x00, sizeof(zero)); \ + ZERO_CLONE_ ## which(zero); \ /* Clear entire check buffer for 0xFF overlap test. */ \ memset(check_buf, 0x00, sizeof(check_buf)); \ /* Fill stack with 0xFF. */ \ @@ -139,15 +245,16 @@ static noinline __init int test_ ## name (void) \ for (sum = 0, i = 0; i < target_size; i++) \ sum += (check_buf[i] == 0xFF); \ \ - if (sum == 0) \ + if (sum == 0) { \ pr_info(#name " ok\n"); \ - else \ - pr_warn(#name " FAIL (uninit bytes: %d)\n", \ - sum); \ - \ - return (sum != 0); \ + return 0; \ + } else { \ + pr_warn(#name " %sFAIL (uninit bytes: %d)\n", \ + (xfail) ? "X" : "", sum); \ + return (xfail) ? 0 : 1; \ + } \ } -#define DEFINE_TEST(name, var_type, which, init_level) \ +#define DEFINE_TEST(name, var_type, which, init_level, xfail) \ /* no-op to force compiler into ignoring "uninitialized" vars */\ static noinline __init DO_NOTHING_TYPE_ ## which(var_type) \ do_nothing_ ## name(var_type *ptr) \ @@ -163,7 +270,8 @@ static noinline __init int leaf_ ## name(unsigned long sp, \ var_type *arg) \ { \ char buf[VAR_BUFFER]; \ - var_type var INIT_ ## which ## _ ## init_level; \ + var_type var \ + INIT_ ## which ## _ ## init_level(var_type); \ \ target_start = &var; \ target_size = sizeof(var); \ @@ -189,7 +297,7 @@ static noinline __init int leaf_ ## name(unsigned long sp, \ \ return (int)buf[0] | (int)buf[sizeof(buf) - 1]; \ } \ -DEFINE_TEST_DRIVER(name, var_type, which) +DEFINE_TEST_DRIVER(name, var_type, which, xfail) /* Structure with no padding. */ struct test_packed { @@ -208,18 +316,13 @@ struct test_small_hole { unsigned long four; }; -/* Try to trigger unhandled padding in a structure. */ -struct test_aligned { - u32 internal1; - u64 internal2; -} __aligned(64); - +/* Trigger unhandled padding in a structure. */ struct test_big_hole { u8 one; u8 two; u8 three; /* 61 byte padding hole here. */ - struct test_aligned four; + u8 four __aligned(64); } __aligned(64); struct test_trailing_hole { @@ -238,42 +341,50 @@ struct test_user { unsigned long four; }; -#define DEFINE_SCALAR_TEST(name, init) \ - DEFINE_TEST(name ## _ ## init, name, SCALAR, init) +#define DEFINE_SCALAR_TEST(name, init, xfail) \ + DEFINE_TEST(name ## _ ## init, name, SCALAR, \ + init, xfail) -#define DEFINE_SCALAR_TESTS(init) \ - DEFINE_SCALAR_TEST(u8, init); \ - DEFINE_SCALAR_TEST(u16, init); \ - DEFINE_SCALAR_TEST(u32, init); \ - DEFINE_SCALAR_TEST(u64, init); \ - DEFINE_TEST(char_array_ ## init, unsigned char, STRING, init) +#define DEFINE_SCALAR_TESTS(init, xfail) \ + DEFINE_SCALAR_TEST(u8, init, xfail); \ + DEFINE_SCALAR_TEST(u16, init, xfail); \ + DEFINE_SCALAR_TEST(u32, init, xfail); \ + DEFINE_SCALAR_TEST(u64, init, xfail); \ + DEFINE_TEST(char_array_ ## init, unsigned char, \ + STRING, init, xfail) -#define DEFINE_STRUCT_TEST(name, init) \ +#define DEFINE_STRUCT_TEST(name, init, xfail) \ DEFINE_TEST(name ## _ ## init, \ - struct test_ ## name, STRUCT, init) + struct test_ ## name, STRUCT, init, \ + xfail) + +#define DEFINE_STRUCT_TESTS(init, xfail) \ + DEFINE_STRUCT_TEST(small_hole, init, xfail); \ + DEFINE_STRUCT_TEST(big_hole, init, xfail); \ + DEFINE_STRUCT_TEST(trailing_hole, init, xfail); \ + DEFINE_STRUCT_TEST(packed, init, xfail) -#define DEFINE_STRUCT_TESTS(init) \ - DEFINE_STRUCT_TEST(small_hole, init); \ - DEFINE_STRUCT_TEST(big_hole, init); \ - DEFINE_STRUCT_TEST(trailing_hole, init); \ - DEFINE_STRUCT_TEST(packed, init) +#define DEFINE_STRUCT_INITIALIZER_TESTS(base) \ + DEFINE_STRUCT_TESTS(base ## _ ## partial, \ + WANT_SUCCESS); \ + DEFINE_STRUCT_TESTS(base ## _ ## all, \ + WANT_SUCCESS) /* These should be fully initialized all the time! */ -DEFINE_SCALAR_TESTS(zero); -DEFINE_STRUCT_TESTS(zero); -/* Static initialization: padding may be left uninitialized. */ -DEFINE_STRUCT_TESTS(static_partial); -DEFINE_STRUCT_TESTS(static_all); -/* Dynamic initialization: padding may be left uninitialized. */ -DEFINE_STRUCT_TESTS(dynamic_partial); -DEFINE_STRUCT_TESTS(dynamic_all); -/* Runtime initialization: padding may be left uninitialized. */ -DEFINE_STRUCT_TESTS(runtime_partial); -DEFINE_STRUCT_TESTS(runtime_all); +DEFINE_SCALAR_TESTS(zero, WANT_SUCCESS); +DEFINE_STRUCT_TESTS(zero, WANT_SUCCESS); +/* Struct initializers: padding may be left uninitialized. */ +DEFINE_STRUCT_INITIALIZER_TESTS(static); +DEFINE_STRUCT_INITIALIZER_TESTS(dynamic); +DEFINE_STRUCT_INITIALIZER_TESTS(runtime); +DEFINE_STRUCT_INITIALIZER_TESTS(assigned_static); +DEFINE_STRUCT_INITIALIZER_TESTS(assigned_dynamic); +DEFINE_STRUCT_TESTS(assigned_copy, XFAIL); /* No initialization without compiler instrumentation. */ -DEFINE_SCALAR_TESTS(none); -DEFINE_STRUCT_TESTS(none); -DEFINE_TEST(user, struct test_user, STRUCT, none); +DEFINE_SCALAR_TESTS(none, WANT_SUCCESS); +DEFINE_STRUCT_TESTS(none, WANT_SUCCESS); +/* Initialization of members with __user attribute. */ +DEFINE_TEST(user, struct test_user, STRUCT, none, WANT_SUCCESS); /* * Check two uses through a variable declaration outside either path, @@ -283,6 +394,10 @@ DEFINE_TEST(user, struct test_user, STRUCT, none); static int noinline __leaf_switch_none(int path, bool fill) { switch (path) { + /* + * This is intentionally unreachable. To silence the + * warning, build with -Wno-switch-unreachable + */ uint64_t var; case 1: @@ -326,8 +441,14 @@ static noinline __init int leaf_switch_2_none(unsigned long sp, bool fill, return __leaf_switch_none(2, fill); } -DEFINE_TEST_DRIVER(switch_1_none, uint64_t, SCALAR); -DEFINE_TEST_DRIVER(switch_2_none, uint64_t, SCALAR); +/* + * These are expected to fail for most configurations because neither + * GCC nor Clang have a way to perform initialization of variables in + * non-code areas (i.e. in a switch statement before the first "case"). + * https://bugs.llvm.org/show_bug.cgi?id=44916 + */ +DEFINE_TEST_DRIVER(switch_1_none, uint64_t, SCALAR, XFAIL); +DEFINE_TEST_DRIVER(switch_2_none, uint64_t, SCALAR, XFAIL); static int __init test_stackinit_init(void) { @@ -353,12 +474,18 @@ static int __init test_stackinit_init(void) test_structs(zero); /* Padding here appears to be accidentally always initialized? */ test_structs(dynamic_partial); + test_structs(assigned_dynamic_partial); /* Padding initialization depends on compiler behaviors. */ test_structs(static_partial); test_structs(static_all); test_structs(dynamic_all); test_structs(runtime_partial); test_structs(runtime_all); + test_structs(assigned_static_partial); + test_structs(assigned_static_all); + test_structs(assigned_dynamic_all); + /* Everything fails this since it effectively performs a memcpy(). */ + test_structs(assigned_copy); /* STRUCTLEAK_BYREF_ALL should cover everything from here down. */ test_scalars(none); diff --git a/lib/test_string.c b/lib/test_string.c index 7b31f4a505bf..9dfd6f52de92 100644 --- a/lib/test_string.c +++ b/lib/test_string.c @@ -179,6 +179,10 @@ static __init int strnchr_selftest(void) return 0; } +static __exit void string_selftest_remove(void) +{ +} + static __init int string_selftest_init(void) { int test, subtest; @@ -216,4 +220,5 @@ fail: } module_init(string_selftest_init); +module_exit(string_selftest_remove); MODULE_LICENSE("GPL v2"); diff --git a/lib/test_sysctl.c b/lib/test_sysctl.c index 566dad3f4196..3750323973f4 100644 --- a/lib/test_sysctl.c +++ b/lib/test_sysctl.c @@ -16,7 +16,7 @@ */ /* - * This module provides an interface to the the proc sysctl interfaces. This + * This module provides an interface to the proc sysctl interfaces. This * driver requires CONFIG_PROC_SYSCTL. It will not normally be loaded by the * system unless explicitly requested by name. You can also build this driver * into your kernel. @@ -44,6 +44,8 @@ struct test_sysctl_data { int int_0002; int int_0003[4]; + int boot_int; + unsigned int uint_0001; char string_0001[65]; @@ -61,6 +63,8 @@ static struct test_sysctl_data test_data = { .int_0003[2] = 2, .int_0003[3] = 3, + .boot_int = 0, + .uint_0001 = 314, .string_0001 = "(none)", @@ -92,6 +96,15 @@ static struct ctl_table test_table[] = { .proc_handler = proc_dointvec, }, { + .procname = "boot_int", + .data = &test_data.boot_int, + .maxlen = sizeof(test_data.boot_int), + .mode = 0644, + .proc_handler = proc_dointvec, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { .procname = "uint_0001", .data = &test_data.uint_0001, .maxlen = sizeof(unsigned int), @@ -149,7 +162,7 @@ static int __init test_sysctl_init(void) } return 0; } -late_initcall(test_sysctl_init); +module_init(test_sysctl_init); static void __exit test_sysctl_exit(void) { diff --git a/lib/test_ubsan.c b/lib/test_ubsan.c index 9ea10adf7a66..7e7bbd0f3fd2 100644 --- a/lib/test_ubsan.c +++ b/lib/test_ubsan.c @@ -5,71 +5,78 @@ typedef void(*test_ubsan_fp)(void); -static void test_ubsan_add_overflow(void) -{ - volatile int val = INT_MAX; - - val += 2; -} - -static void test_ubsan_sub_overflow(void) -{ - volatile int val = INT_MIN; - volatile int val2 = 2; - - val -= val2; -} - -static void test_ubsan_mul_overflow(void) -{ - volatile int val = INT_MAX / 2; - - val *= 3; -} - -static void test_ubsan_negate_overflow(void) -{ - volatile int val = INT_MIN; - - val = -val; -} +#define UBSAN_TEST(config, ...) do { \ + pr_info("%s " __VA_ARGS__ "%s(%s=%s)\n", __func__, \ + sizeof(" " __VA_ARGS__) > 2 ? " " : "", \ + #config, IS_ENABLED(config) ? "y" : "n"); \ + } while (0) static void test_ubsan_divrem_overflow(void) { volatile int val = 16; volatile int val2 = 0; + UBSAN_TEST(CONFIG_UBSAN_DIV_ZERO); val /= val2; } static void test_ubsan_shift_out_of_bounds(void) { - volatile int val = -1; - int val2 = 10; + volatile int neg = -1, wrap = 4; + int val1 = 10; + int val2 = INT_MAX; + + UBSAN_TEST(CONFIG_UBSAN_SHIFT, "negative exponent"); + val1 <<= neg; - val2 <<= val; + UBSAN_TEST(CONFIG_UBSAN_SHIFT, "left overflow"); + val2 <<= wrap; } static void test_ubsan_out_of_bounds(void) { - volatile int i = 4, j = 5; + volatile int i = 4, j = 5, k = -1; + volatile char above[4] = { }; /* Protect surrounding memory. */ volatile int arr[4]; + volatile char below[4] = { }; /* Protect surrounding memory. */ + above[0] = below[0]; + + UBSAN_TEST(CONFIG_UBSAN_BOUNDS, "above"); arr[j] = i; + + UBSAN_TEST(CONFIG_UBSAN_BOUNDS, "below"); + arr[k] = i; } +enum ubsan_test_enum { + UBSAN_TEST_ZERO = 0, + UBSAN_TEST_ONE, + UBSAN_TEST_MAX, +}; + static void test_ubsan_load_invalid_value(void) { volatile char *dst, *src; bool val, val2, *ptr; - char c = 4; + enum ubsan_test_enum eval, eval2, *eptr; + unsigned char c = 0xff; + UBSAN_TEST(CONFIG_UBSAN_BOOL, "bool"); dst = (char *)&val; src = &c; *dst = *src; ptr = &val2; val2 = val; + + UBSAN_TEST(CONFIG_UBSAN_ENUM, "enum"); + dst = (char *)&eval; + src = &c; + *dst = *src; + + eptr = &eval2; + eval2 = eval; } static void test_ubsan_null_ptr_deref(void) @@ -77,6 +84,7 @@ static void test_ubsan_null_ptr_deref(void) volatile int *ptr = NULL; int val; + UBSAN_TEST(CONFIG_UBSAN_OBJECT_SIZE); val = *ptr; } @@ -85,6 +93,7 @@ static void test_ubsan_misaligned_access(void) volatile char arr[5] __aligned(4) = {1, 2, 3, 4, 5}; volatile int *ptr, val = 6; + UBSAN_TEST(CONFIG_UBSAN_ALIGNMENT); ptr = (int *)(arr + 1); *ptr = val; } @@ -95,24 +104,25 @@ static void test_ubsan_object_size_mismatch(void) volatile int val __aligned(8) = 4; volatile long long *ptr, val2; + UBSAN_TEST(CONFIG_UBSAN_OBJECT_SIZE); ptr = (long long *)&val; val2 = *ptr; } static const test_ubsan_fp test_ubsan_array[] = { - test_ubsan_add_overflow, - test_ubsan_sub_overflow, - test_ubsan_mul_overflow, - test_ubsan_negate_overflow, - test_ubsan_divrem_overflow, test_ubsan_shift_out_of_bounds, test_ubsan_out_of_bounds, test_ubsan_load_invalid_value, - //test_ubsan_null_ptr_deref, /* exclude it because there is a crash */ test_ubsan_misaligned_access, test_ubsan_object_size_mismatch, }; +/* Excluded because they Oops the module. */ +static const test_ubsan_fp skip_ubsan_array[] = { + test_ubsan_divrem_overflow, + test_ubsan_null_ptr_deref, +}; + static int __init test_ubsan_init(void) { unsigned int i; @@ -120,7 +130,6 @@ static int __init test_ubsan_init(void) for (i = 0; i < ARRAY_SIZE(test_ubsan_array); i++) test_ubsan_array[i](); - (void)test_ubsan_null_ptr_deref; /* to avoid unsed-function warning */ return 0; } module_init(test_ubsan_init); diff --git a/lib/test_vmalloc.c b/lib/test_vmalloc.c index 8bbefcaddfe8..cf41fd6df42a 100644 --- a/lib/test_vmalloc.c +++ b/lib/test_vmalloc.c @@ -15,14 +15,16 @@ #include <linux/delay.h> #include <linux/rwsem.h> #include <linux/mm.h> +#include <linux/rcupdate.h> +#include <linux/slab.h> #define __param(type, name, init, msg) \ static type name = init; \ module_param(name, type, 0444); \ MODULE_PARM_DESC(name, msg) \ -__param(bool, single_cpu_test, false, - "Use single first online CPU to run tests"); +__param(int, nr_threads, 0, + "Number of workers to perform tests(min: 1 max: USHRT_MAX)"); __param(bool, sequential_test_order, false, "Use sequential stress tests order"); @@ -33,27 +35,25 @@ __param(int, test_repeat_count, 1, __param(int, test_loop_count, 1000000, "Set test loop counter"); +__param(int, nr_pages, 0, + "Set number of pages for fix_size_alloc_test(default: 1)"); + __param(int, run_test_mask, INT_MAX, "Set tests specified in the mask.\n\n" - "\t\tid: 1, name: fix_size_alloc_test\n" - "\t\tid: 2, name: full_fit_alloc_test\n" - "\t\tid: 4, name: long_busy_list_alloc_test\n" - "\t\tid: 8, name: random_size_alloc_test\n" - "\t\tid: 16, name: fix_align_alloc_test\n" - "\t\tid: 32, name: random_size_align_alloc_test\n" - "\t\tid: 64, name: align_shift_alloc_test\n" - "\t\tid: 128, name: pcpu_alloc_test\n" + "\t\tid: 1, name: fix_size_alloc_test\n" + "\t\tid: 2, name: full_fit_alloc_test\n" + "\t\tid: 4, name: long_busy_list_alloc_test\n" + "\t\tid: 8, name: random_size_alloc_test\n" + "\t\tid: 16, name: fix_align_alloc_test\n" + "\t\tid: 32, name: random_size_align_alloc_test\n" + "\t\tid: 64, name: align_shift_alloc_test\n" + "\t\tid: 128, name: pcpu_alloc_test\n" + "\t\tid: 256, name: kvfree_rcu_1_arg_vmalloc_test\n" + "\t\tid: 512, name: kvfree_rcu_2_arg_vmalloc_test\n" /* Add a new test case description here. */ ); /* - * Depends on single_cpu_test parameter. If it is true, then - * use first online CPU to trigger a test on, otherwise go with - * all online CPUs. - */ -static cpumask_t cpus_run_test_mask = CPU_MASK_NONE; - -/* * Read write semaphore for synchronization of setup * phase that is done in main thread and workers. */ @@ -91,12 +91,8 @@ static int random_size_align_alloc_test(void) */ size = ((rnd % 10) + 1) * PAGE_SIZE; - ptr = __vmalloc_node_range(size, align, - VMALLOC_START, VMALLOC_END, - GFP_KERNEL | __GFP_ZERO, - PAGE_KERNEL, - 0, 0, __builtin_return_address(0)); - + ptr = __vmalloc_node(size, align, GFP_KERNEL | __GFP_ZERO, 0, + __builtin_return_address(0)); if (!ptr) return -1; @@ -118,12 +114,8 @@ static int align_shift_alloc_test(void) for (i = 0; i < BITS_PER_LONG; i++) { align = ((unsigned long) 1) << i; - ptr = __vmalloc_node_range(PAGE_SIZE, align, - VMALLOC_START, VMALLOC_END, - GFP_KERNEL | __GFP_ZERO, - PAGE_KERNEL, - 0, 0, __builtin_return_address(0)); - + ptr = __vmalloc_node(PAGE_SIZE, align, GFP_KERNEL|__GFP_ZERO, 0, + __builtin_return_address(0)); if (!ptr) return -1; @@ -139,13 +131,9 @@ static int fix_align_alloc_test(void) int i; for (i = 0; i < test_loop_count; i++) { - ptr = __vmalloc_node_range(5 * PAGE_SIZE, - THREAD_ALIGN << 1, - VMALLOC_START, VMALLOC_END, - GFP_KERNEL | __GFP_ZERO, - PAGE_KERNEL, - 0, 0, __builtin_return_address(0)); - + ptr = __vmalloc_node(5 * PAGE_SIZE, THREAD_ALIGN << 1, + GFP_KERNEL | __GFP_ZERO, 0, + __builtin_return_address(0)); if (!ptr) return -1; @@ -277,7 +265,7 @@ static int fix_size_alloc_test(void) int i; for (i = 0; i < test_loop_count; i++) { - ptr = vmalloc(3 * PAGE_SIZE); + ptr = vmalloc((nr_pages > 0 ? nr_pages:1) * PAGE_SIZE); if (!ptr) return -1; @@ -328,6 +316,47 @@ pcpu_alloc_test(void) return rv; } +struct test_kvfree_rcu { + struct rcu_head rcu; + unsigned char array[20]; +}; + +static int +kvfree_rcu_1_arg_vmalloc_test(void) +{ + struct test_kvfree_rcu *p; + int i; + + for (i = 0; i < test_loop_count; i++) { + p = vmalloc(1 * PAGE_SIZE); + if (!p) + return -1; + + p->array[0] = 'a'; + kvfree_rcu(p); + } + + return 0; +} + +static int +kvfree_rcu_2_arg_vmalloc_test(void) +{ + struct test_kvfree_rcu *p; + int i; + + for (i = 0; i < test_loop_count; i++) { + p = vmalloc(1 * PAGE_SIZE); + if (!p) + return -1; + + p->array[0] = 'a'; + kvfree_rcu(p, rcu); + } + + return 0; +} + struct test_case_desc { const char *test_name; int (*test_func)(void); @@ -342,6 +371,8 @@ static struct test_case_desc test_case_array[] = { { "random_size_align_alloc_test", random_size_align_alloc_test }, { "align_shift_alloc_test", align_shift_alloc_test }, { "pcpu_alloc_test", pcpu_alloc_test }, + { "kvfree_rcu_1_arg_vmalloc_test", kvfree_rcu_1_arg_vmalloc_test }, + { "kvfree_rcu_2_arg_vmalloc_test", kvfree_rcu_2_arg_vmalloc_test }, /* Add a new test case here. */ }; @@ -351,21 +382,18 @@ struct test_case_data { u64 time; }; -/* Split it to get rid of: WARNING: line over 80 characters */ -static struct test_case_data - per_cpu_test_data[NR_CPUS][ARRAY_SIZE(test_case_array)]; - static struct test_driver { struct task_struct *task; + struct test_case_data data[ARRAY_SIZE(test_case_array)]; + unsigned long start; unsigned long stop; - int cpu; -} per_cpu_test_driver[NR_CPUS]; +} *tdriver; static void shuffle_array(int *arr, int n) { unsigned int rnd; - int i, j, x; + int i, j; for (i = n - 1; i > 0; i--) { get_random_bytes(&rnd, sizeof(rnd)); @@ -374,9 +402,7 @@ static void shuffle_array(int *arr, int n) j = rnd % i; /* Swap indexes. */ - x = arr[i]; - arr[i] = arr[j]; - arr[j] = x; + swap(arr[i], arr[j]); } } @@ -388,9 +414,6 @@ static int test_func(void *private) ktime_t kt; u64 delta; - if (set_cpus_allowed_ptr(current, cpumask_of(t->cpu)) < 0) - pr_err("Failed to set affinity to %d CPU\n", t->cpu); - for (i = 0; i < ARRAY_SIZE(test_case_array); i++) random_array[i] = i; @@ -415,9 +438,9 @@ static int test_func(void *private) kt = ktime_get(); for (j = 0; j < test_repeat_count; j++) { if (!test_case_array[index].test_func()) - per_cpu_test_data[t->cpu][index].test_passed++; + t->data[index].test_passed++; else - per_cpu_test_data[t->cpu][index].test_failed++; + t->data[index].test_failed++; } /* @@ -426,7 +449,7 @@ static int test_func(void *private) delta = (u64) ktime_us_delta(ktime_get(), kt); do_div(delta, (u32) test_repeat_count); - per_cpu_test_data[t->cpu][index].time = delta; + t->data[index].time = delta; } t->stop = get_cycles(); @@ -442,53 +465,56 @@ static int test_func(void *private) return 0; } -static void +static int init_test_configurtion(void) { /* - * Reset all data of all CPUs. + * A maximum number of workers is defined as hard-coded + * value and set to USHRT_MAX. We add such gap just in + * case and for potential heavy stressing. */ - memset(per_cpu_test_data, 0, sizeof(per_cpu_test_data)); + nr_threads = clamp(nr_threads, 1, (int) USHRT_MAX); - if (single_cpu_test) - cpumask_set_cpu(cpumask_first(cpu_online_mask), - &cpus_run_test_mask); - else - cpumask_and(&cpus_run_test_mask, cpu_online_mask, - cpu_online_mask); + /* Allocate the space for test instances. */ + tdriver = kvcalloc(nr_threads, sizeof(*tdriver), GFP_KERNEL); + if (tdriver == NULL) + return -1; if (test_repeat_count <= 0) test_repeat_count = 1; if (test_loop_count <= 0) test_loop_count = 1; + + return 0; } static void do_concurrent_test(void) { - int cpu, ret; + int i, ret; /* * Set some basic configurations plus sanity check. */ - init_test_configurtion(); + ret = init_test_configurtion(); + if (ret < 0) + return; /* * Put on hold all workers. */ down_write(&prepare_for_test_rwsem); - for_each_cpu(cpu, &cpus_run_test_mask) { - struct test_driver *t = &per_cpu_test_driver[cpu]; + for (i = 0; i < nr_threads; i++) { + struct test_driver *t = &tdriver[i]; - t->cpu = cpu; - t->task = kthread_run(test_func, t, "vmalloc_test/%d", cpu); + t->task = kthread_run(test_func, t, "vmalloc_test/%d", i); if (!IS_ERR(t->task)) /* Success. */ atomic_inc(&test_n_undone); else - pr_err("Failed to start kthread for %d CPU\n", cpu); + pr_err("Failed to start %d kthread\n", i); } /* @@ -506,29 +532,31 @@ static void do_concurrent_test(void) ret = wait_for_completion_timeout(&test_all_done_comp, HZ); } while (!ret); - for_each_cpu(cpu, &cpus_run_test_mask) { - struct test_driver *t = &per_cpu_test_driver[cpu]; - int i; + for (i = 0; i < nr_threads; i++) { + struct test_driver *t = &tdriver[i]; + int j; if (!IS_ERR(t->task)) kthread_stop(t->task); - for (i = 0; i < ARRAY_SIZE(test_case_array); i++) { - if (!((run_test_mask & (1 << i)) >> i)) + for (j = 0; j < ARRAY_SIZE(test_case_array); j++) { + if (!((run_test_mask & (1 << j)) >> j)) continue; pr_info( "Summary: %s passed: %d failed: %d repeat: %d loops: %d avg: %llu usec\n", - test_case_array[i].test_name, - per_cpu_test_data[cpu][i].test_passed, - per_cpu_test_data[cpu][i].test_failed, + test_case_array[j].test_name, + t->data[j].test_passed, + t->data[j].test_failed, test_repeat_count, test_loop_count, - per_cpu_test_data[cpu][i].time); + t->data[j].time); } - pr_info("All test took CPU%d=%lu cycles\n", - cpu, t->stop - t->start); + pr_info("All test took worker%d=%lu cycles\n", + i, t->stop - t->start); } + + kvfree(tdriver); } static int vmalloc_test_init(void) diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 7df4f7f395bf..8b1c318189ce 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -2,6 +2,7 @@ /* * test_xarray.c: Test the XArray API * Copyright (c) 2017-2018 Microsoft Corporation + * Copyright (c) 2019-2020 Oracle * Author: Matthew Wilcox <willy@infradead.org> */ @@ -11,6 +12,9 @@ static unsigned int tests_run; static unsigned int tests_passed; +static const unsigned int order_limit = + IS_ENABLED(CONFIG_XARRAY_MULTI) ? BITS_PER_LONG : 1; + #ifndef XA_DEBUG # ifdef __KERNEL__ void xa_dump(const struct xarray *xa) { } @@ -285,6 +289,27 @@ static noinline void check_xa_mark_2(struct xarray *xa) xa_destroy(xa); } +static noinline void check_xa_mark_3(struct xarray *xa) +{ +#ifdef CONFIG_XARRAY_MULTI + XA_STATE(xas, xa, 0x41); + void *entry; + int count = 0; + + xa_store_order(xa, 0x40, 2, xa_mk_index(0x40), GFP_KERNEL); + xa_set_mark(xa, 0x41, XA_MARK_0); + + rcu_read_lock(); + xas_for_each_marked(&xas, entry, ULONG_MAX, XA_MARK_0) { + count++; + XA_BUG_ON(xa, entry != xa_mk_index(0x40)); + } + XA_BUG_ON(xa, count != 1); + rcu_read_unlock(); + xa_destroy(xa); +#endif +} + static noinline void check_xa_mark(struct xarray *xa) { unsigned long index; @@ -293,6 +318,7 @@ static noinline void check_xa_mark(struct xarray *xa) check_xa_mark_1(xa, index); check_xa_mark_2(xa); + check_xa_mark_3(xa); } static noinline void check_xa_shrink(struct xarray *xa) @@ -389,6 +415,9 @@ static noinline void check_cmpxchg(struct xarray *xa) XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE); XA_BUG_ON(xa, xa_cmpxchg(xa, 5, FIVE, NULL, GFP_KERNEL) != NULL); XA_BUG_ON(xa, xa_cmpxchg(xa, 5, NULL, FIVE, GFP_KERNEL) != NULL); + XA_BUG_ON(xa, xa_insert(xa, 5, FIVE, GFP_KERNEL) != -EBUSY); + XA_BUG_ON(xa, xa_cmpxchg(xa, 5, FIVE, NULL, GFP_KERNEL) != FIVE); + XA_BUG_ON(xa, xa_insert(xa, 5, FIVE, GFP_KERNEL) == -EBUSY); xa_erase_index(xa, 12345678); xa_erase_index(xa, 5); XA_BUG_ON(xa, !xa_empty(xa)); @@ -902,28 +931,34 @@ static noinline void check_store_iter(struct xarray *xa) XA_BUG_ON(xa, !xa_empty(xa)); } -static noinline void check_multi_find(struct xarray *xa) +static noinline void check_multi_find_1(struct xarray *xa, unsigned order) { #ifdef CONFIG_XARRAY_MULTI + unsigned long multi = 3 << order; + unsigned long next = 4 << order; unsigned long index; - xa_store_order(xa, 12, 2, xa_mk_value(12), GFP_KERNEL); - XA_BUG_ON(xa, xa_store_index(xa, 16, GFP_KERNEL) != NULL); + xa_store_order(xa, multi, order, xa_mk_value(multi), GFP_KERNEL); + XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL); + XA_BUG_ON(xa, xa_store_index(xa, next + 1, GFP_KERNEL) != NULL); index = 0; XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) != - xa_mk_value(12)); - XA_BUG_ON(xa, index != 12); - index = 13; + xa_mk_value(multi)); + XA_BUG_ON(xa, index != multi); + index = multi + 1; XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) != - xa_mk_value(12)); - XA_BUG_ON(xa, (index < 12) || (index >= 16)); + xa_mk_value(multi)); + XA_BUG_ON(xa, (index < multi) || (index >= next)); XA_BUG_ON(xa, xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT) != - xa_mk_value(16)); - XA_BUG_ON(xa, index != 16); - - xa_erase_index(xa, 12); - xa_erase_index(xa, 16); + xa_mk_value(next)); + XA_BUG_ON(xa, index != next); + XA_BUG_ON(xa, xa_find_after(xa, &index, next, XA_PRESENT) != NULL); + XA_BUG_ON(xa, index != next); + + xa_erase_index(xa, multi); + xa_erase_index(xa, next); + xa_erase_index(xa, next + 1); XA_BUG_ON(xa, !xa_empty(xa)); #endif } @@ -952,6 +987,20 @@ static noinline void check_multi_find_2(struct xarray *xa) } } +static noinline void check_multi_find_3(struct xarray *xa) +{ + unsigned int order; + + for (order = 5; order < order_limit; order++) { + unsigned long index = 1UL << (order - 5); + + XA_BUG_ON(xa, !xa_empty(xa)); + xa_store_order(xa, 0, order - 4, xa_mk_index(0), GFP_KERNEL); + XA_BUG_ON(xa, xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT)); + xa_erase_index(xa, 0); + } +} + static noinline void check_find_1(struct xarray *xa) { unsigned long i, j, k; @@ -1046,13 +1095,35 @@ static noinline void check_find_3(struct xarray *xa) xa_destroy(xa); } +static noinline void check_find_4(struct xarray *xa) +{ + unsigned long index = 0; + void *entry; + + xa_store_index(xa, ULONG_MAX, GFP_KERNEL); + + entry = xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT); + XA_BUG_ON(xa, entry != xa_mk_index(ULONG_MAX)); + + entry = xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT); + XA_BUG_ON(xa, entry); + + xa_erase_index(xa, ULONG_MAX); +} + static noinline void check_find(struct xarray *xa) { + unsigned i; + check_find_1(xa); check_find_2(xa); check_find_3(xa); - check_multi_find(xa); + check_find_4(xa); + + for (i = 2; i < 10; i++) + check_multi_find_1(xa, i); check_multi_find_2(xa); + check_multi_find_3(xa); } /* See find_swap_entry() in mm/shmem.c */ @@ -1110,6 +1181,42 @@ static noinline void check_find_entry(struct xarray *xa) XA_BUG_ON(xa, !xa_empty(xa)); } +static noinline void check_pause(struct xarray *xa) +{ + XA_STATE(xas, xa, 0); + void *entry; + unsigned int order; + unsigned long index = 1; + unsigned int count = 0; + + for (order = 0; order < order_limit; order++) { + XA_BUG_ON(xa, xa_store_order(xa, index, order, + xa_mk_index(index), GFP_KERNEL)); + index += 1UL << order; + } + + rcu_read_lock(); + xas_for_each(&xas, entry, ULONG_MAX) { + XA_BUG_ON(xa, entry != xa_mk_index(1UL << count)); + count++; + } + rcu_read_unlock(); + XA_BUG_ON(xa, count != order_limit); + + count = 0; + xas_set(&xas, 0); + rcu_read_lock(); + xas_for_each(&xas, entry, ULONG_MAX) { + XA_BUG_ON(xa, entry != xa_mk_index(1UL << count)); + count++; + xas_pause(&xas); + } + rcu_read_unlock(); + XA_BUG_ON(xa, count != order_limit); + + xa_destroy(xa); +} + static noinline void check_move_tiny(struct xarray *xa) { XA_STATE(xas, xa, 0); @@ -1132,6 +1239,27 @@ static noinline void check_move_tiny(struct xarray *xa) XA_BUG_ON(xa, !xa_empty(xa)); } +static noinline void check_move_max(struct xarray *xa) +{ + XA_STATE(xas, xa, 0); + + xa_store_index(xa, ULONG_MAX, GFP_KERNEL); + rcu_read_lock(); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_index(ULONG_MAX)); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != NULL); + rcu_read_unlock(); + + xas_set(&xas, 0); + rcu_read_lock(); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_index(ULONG_MAX)); + xas_pause(&xas); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != NULL); + rcu_read_unlock(); + + xa_erase_index(xa, ULONG_MAX); + XA_BUG_ON(xa, !xa_empty(xa)); +} + static noinline void check_move_small(struct xarray *xa, unsigned long idx) { XA_STATE(xas, xa, 0); @@ -1240,6 +1368,7 @@ static noinline void check_move(struct xarray *xa) xa_destroy(xa); check_move_tiny(xa); + check_move_max(xa); for (i = 0; i < 16; i++) check_move_small(xa, 1UL << i); @@ -1399,6 +1528,51 @@ static noinline void check_store_range(struct xarray *xa) } } +#ifdef CONFIG_XARRAY_MULTI +static void check_split_1(struct xarray *xa, unsigned long index, + unsigned int order, unsigned int new_order) +{ + XA_STATE_ORDER(xas, xa, index, new_order); + unsigned int i; + + xa_store_order(xa, index, order, xa, GFP_KERNEL); + + xas_split_alloc(&xas, xa, order, GFP_KERNEL); + xas_lock(&xas); + xas_split(&xas, xa, order); + for (i = 0; i < (1 << order); i += (1 << new_order)) + __xa_store(xa, index + i, xa_mk_index(index + i), 0); + xas_unlock(&xas); + + for (i = 0; i < (1 << order); i++) { + unsigned int val = index + (i & ~((1 << new_order) - 1)); + XA_BUG_ON(xa, xa_load(xa, index + i) != xa_mk_index(val)); + } + + xa_set_mark(xa, index, XA_MARK_0); + XA_BUG_ON(xa, !xa_get_mark(xa, index, XA_MARK_0)); + + xa_destroy(xa); +} + +static noinline void check_split(struct xarray *xa) +{ + unsigned int order, new_order; + + XA_BUG_ON(xa, !xa_empty(xa)); + + for (order = 1; order < 2 * XA_CHUNK_SHIFT; order++) { + for (new_order = 0; new_order < order; new_order++) { + check_split_1(xa, 0, order, new_order); + check_split_1(xa, 1UL << order, order, new_order); + check_split_1(xa, 3UL << order, order, new_order); + } + } +} +#else +static void check_split(struct xarray *xa) { } +#endif + static void check_align_1(struct xarray *xa, char *name) { int i; @@ -1471,14 +1645,9 @@ static noinline void shadow_remove(struct xarray *xa) xa_lock(xa); while ((node = list_first_entry_or_null(&shadow_nodes, struct xa_node, private_list))) { - XA_STATE(xas, node->array, 0); XA_BUG_ON(xa, node->array != xa); list_del_init(&node->private_list); - xas.xa_node = xa_parent_locked(node->array, node); - xas.xa_offset = node->offset; - xas.xa_shift = node->shift + XA_CHUNK_SHIFT; - xas_set_update(&xas, test_update_node); - xas_store(&xas, NULL); + xa_delete_node(node, test_update_node); } xa_unlock(xa); } @@ -1545,6 +1714,26 @@ static noinline void check_account(struct xarray *xa) #endif } +static noinline void check_get_order(struct xarray *xa) +{ + unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 20 : 1; + unsigned int order; + unsigned long i, j; + + for (i = 0; i < 3; i++) + XA_BUG_ON(xa, xa_get_order(xa, i) != 0); + + for (order = 0; order < max_order; order++) { + for (i = 0; i < 10; i++) { + xa_store_order(xa, i << order, order, + xa_mk_index(i << order), GFP_KERNEL); + for (j = i << order; j < (i + 1) << order; j++) + XA_BUG_ON(xa, xa_get_order(xa, j) != order); + xa_erase(xa, i << order); + } + } +} + static noinline void check_destroy(struct xarray *xa) { unsigned long index; @@ -1593,9 +1782,11 @@ static int xarray_checks(void) check_reserve(&array); check_reserve(&xa0); check_multi_store(&array); + check_get_order(&array); check_xa_alloc(); check_find(&array); check_find_entry(&array); + check_pause(&array); check_account(&array); check_destroy(&array); check_move(&array); @@ -1603,6 +1794,7 @@ static int xarray_checks(void) check_store_range(&array); check_store_iter(&array); check_align(&xa0); + check_split(&array); check_workingset(&array, 0); check_workingset(&array, 64); diff --git a/lib/timerqueue.c b/lib/timerqueue.c index c52710964593..cdb9c7658478 100644 --- a/lib/timerqueue.c +++ b/lib/timerqueue.c @@ -14,6 +14,14 @@ #include <linux/rbtree.h> #include <linux/export.h> +#define __node_2_tq(_n) \ + rb_entry((_n), struct timerqueue_node, node) + +static inline bool __timerqueue_less(struct rb_node *a, const struct rb_node *b) +{ + return __node_2_tq(a)->expires < __node_2_tq(b)->expires; +} + /** * timerqueue_add - Adds timer to timerqueue. * @@ -26,28 +34,10 @@ */ bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node) { - struct rb_node **p = &head->rb_root.rb_root.rb_node; - struct rb_node *parent = NULL; - struct timerqueue_node *ptr; - bool leftmost = true; - /* Make sure we don't add nodes that are already added */ WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node)); - while (*p) { - parent = *p; - ptr = rb_entry(parent, struct timerqueue_node, node); - if (node->expires < ptr->expires) { - p = &(*p)->rb_left; - } else { - p = &(*p)->rb_right; - leftmost = false; - } - } - rb_link_node(&node->node, parent, p); - rb_insert_color_cached(&node->node, &head->rb_root, leftmost); - - return leftmost; + return rb_add_cached(&node->node, &head->rb_root, __timerqueue_less); } EXPORT_SYMBOL_GPL(timerqueue_add); diff --git a/lib/ts_bm.c b/lib/ts_bm.c index b352903c50e3..4cf250031f0f 100644 --- a/lib/ts_bm.c +++ b/lib/ts_bm.c @@ -11,7 +11,7 @@ * [1] A Fast String Searching Algorithm, R.S. Boyer and Moore. * Communications of the Association for Computing Machinery, * 20(10), 1977, pp. 762-772. - * http://www.cs.utexas.edu/users/moore/publications/fstrpos.pdf + * https://www.cs.utexas.edu/users/moore/publications/fstrpos.pdf * * [2] Handbook of Exact String Matching Algorithms, Thierry Lecroq, 2004 * http://www-igm.univ-mlv.fr/~lecroq/string/string.pdf @@ -52,7 +52,7 @@ struct ts_bm u8 * pattern; unsigned int patlen; unsigned int bad_shift[ASIZE]; - unsigned int good_shift[0]; + unsigned int good_shift[]; }; static unsigned int bm_find(struct ts_config *conf, struct ts_state *state) diff --git a/lib/ts_fsm.c b/lib/ts_fsm.c index 9c873cadab7c..64fd9015ad80 100644 --- a/lib/ts_fsm.c +++ b/lib/ts_fsm.c @@ -32,7 +32,7 @@ struct ts_fsm { unsigned int ntokens; - struct ts_fsm_token tokens[0]; + struct ts_fsm_token tokens[]; }; /* other values derived from ctype.h */ @@ -193,7 +193,7 @@ startover: TOKEN_MISMATCH(); block_idx++; - /* fall through */ + fallthrough; case TS_FSM_ANY: if (next == NULL) diff --git a/lib/ts_kmp.c b/lib/ts_kmp.c index 94617e014b3a..c77a3d537f24 100644 --- a/lib/ts_kmp.c +++ b/lib/ts_kmp.c @@ -36,7 +36,7 @@ struct ts_kmp { u8 * pattern; unsigned int pattern_len; - unsigned int prefix_tbl[0]; + unsigned int prefix_tbl[]; }; static unsigned int kmp_find(struct ts_config *conf, struct ts_state *state) diff --git a/lib/ubsan.c b/lib/ubsan.c index 7b9b58aee72c..bdc380ff5d5c 100644 --- a/lib/ubsan.c +++ b/lib/ubsan.c @@ -14,10 +14,11 @@ #include <linux/types.h> #include <linux/sched.h> #include <linux/uaccess.h> +#include <kunit/test-bug.h> #include "ubsan.h" -const char *type_check_kinds[] = { +static const char * const type_check_kinds[] = { "load of", "store to", "reference binding to", @@ -45,13 +46,6 @@ static bool was_reported(struct source_location *location) return test_and_set_bit(REPORTED_BIT, &location->reported); } -static void print_source_location(const char *prefix, - struct source_location *loc) -{ - pr_err("%s %s:%d:%d\n", prefix, loc->file_name, - loc->line & LINE_MASK, loc->column & COLUMN_MASK); -} - static bool suppress_report(struct source_location *loc) { return current->in_ubsan || was_reported(loc); @@ -140,13 +134,16 @@ static void val_to_string(char *str, size_t size, struct type_descriptor *type, } } -static void ubsan_prologue(struct source_location *location) +static void ubsan_prologue(struct source_location *loc, const char *reason) { current->in_ubsan++; pr_err("========================================" "========================================\n"); - print_source_location("UBSAN: Undefined behaviour in", location); + pr_err("UBSAN: %s in %s:%d:%d\n", reason, loc->file_name, + loc->line & LINE_MASK, loc->column & COLUMN_MASK); + + kunit_fail_current_test("%s in %s", reason, loc->file_name); } static void ubsan_epilogue(void) @@ -156,85 +153,28 @@ static void ubsan_epilogue(void) "========================================\n"); current->in_ubsan--; -} - -static void handle_overflow(struct overflow_data *data, void *lhs, - void *rhs, char op) -{ - - struct type_descriptor *type = data->type; - char lhs_val_str[VALUE_LENGTH]; - char rhs_val_str[VALUE_LENGTH]; - - if (suppress_report(&data->location)) - return; - - ubsan_prologue(&data->location); - - val_to_string(lhs_val_str, sizeof(lhs_val_str), type, lhs); - val_to_string(rhs_val_str, sizeof(rhs_val_str), type, rhs); - pr_err("%s integer overflow:\n", - type_is_signed(type) ? "signed" : "unsigned"); - pr_err("%s %c %s cannot be represented in type %s\n", - lhs_val_str, - op, - rhs_val_str, - type->type_name); - - ubsan_epilogue(); -} - -void __ubsan_handle_add_overflow(struct overflow_data *data, - void *lhs, void *rhs) -{ - - handle_overflow(data, lhs, rhs, '+'); -} -EXPORT_SYMBOL(__ubsan_handle_add_overflow); - -void __ubsan_handle_sub_overflow(struct overflow_data *data, - void *lhs, void *rhs) -{ - handle_overflow(data, lhs, rhs, '-'); -} -EXPORT_SYMBOL(__ubsan_handle_sub_overflow); - -void __ubsan_handle_mul_overflow(struct overflow_data *data, - void *lhs, void *rhs) -{ - handle_overflow(data, lhs, rhs, '*'); -} -EXPORT_SYMBOL(__ubsan_handle_mul_overflow); - -void __ubsan_handle_negate_overflow(struct overflow_data *data, - void *old_val) -{ - char old_val_str[VALUE_LENGTH]; - - if (suppress_report(&data->location)) - return; - ubsan_prologue(&data->location); - - val_to_string(old_val_str, sizeof(old_val_str), data->type, old_val); - - pr_err("negation of %s cannot be represented in type %s:\n", - old_val_str, data->type->type_name); - - ubsan_epilogue(); + if (panic_on_warn) { + /* + * This thread may hit another WARN() in the panic path. + * Resetting this prevents additional WARN() from panicking the + * system on this thread. Other threads are blocked by the + * panic_mutex in panic(). + */ + panic_on_warn = 0; + panic("panic_on_warn set ...\n"); + } } -EXPORT_SYMBOL(__ubsan_handle_negate_overflow); - -void __ubsan_handle_divrem_overflow(struct overflow_data *data, - void *lhs, void *rhs) +void __ubsan_handle_divrem_overflow(void *_data, void *lhs, void *rhs) { + struct overflow_data *data = _data; char rhs_val_str[VALUE_LENGTH]; if (suppress_report(&data->location)) return; - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, "division-overflow"); val_to_string(rhs_val_str, sizeof(rhs_val_str), data->type, rhs); @@ -253,7 +193,7 @@ static void handle_null_ptr_deref(struct type_mismatch_data_common *data) if (suppress_report(data->location)) return; - ubsan_prologue(data->location); + ubsan_prologue(data->location, "null-ptr-deref"); pr_err("%s null pointer of type %s\n", type_check_kinds[data->type_check_kind], @@ -268,7 +208,7 @@ static void handle_misaligned_access(struct type_mismatch_data_common *data, if (suppress_report(data->location)) return; - ubsan_prologue(data->location); + ubsan_prologue(data->location, "misaligned-access"); pr_err("%s misaligned address %p for type %s\n", type_check_kinds[data->type_check_kind], @@ -284,7 +224,7 @@ static void handle_object_size_mismatch(struct type_mismatch_data_common *data, if (suppress_report(data->location)) return; - ubsan_prologue(data->location); + ubsan_prologue(data->location, "object-size-mismatch"); pr_err("%s address %p with insufficient space\n", type_check_kinds[data->type_check_kind], (void *) ptr); @@ -321,10 +261,9 @@ void __ubsan_handle_type_mismatch(struct type_mismatch_data *data, } EXPORT_SYMBOL(__ubsan_handle_type_mismatch); -void __ubsan_handle_type_mismatch_v1(struct type_mismatch_data_v1 *data, - void *ptr) +void __ubsan_handle_type_mismatch_v1(void *_data, void *ptr) { - + struct type_mismatch_data_v1 *data = _data; struct type_mismatch_data_common common_data = { .location = &data->location, .type = data->type, @@ -336,14 +275,15 @@ void __ubsan_handle_type_mismatch_v1(struct type_mismatch_data_v1 *data, } EXPORT_SYMBOL(__ubsan_handle_type_mismatch_v1); -void __ubsan_handle_out_of_bounds(struct out_of_bounds_data *data, void *index) +void __ubsan_handle_out_of_bounds(void *_data, void *index) { + struct out_of_bounds_data *data = _data; char index_str[VALUE_LENGTH]; if (suppress_report(&data->location)) return; - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, "array-index-out-of-bounds"); val_to_string(index_str, sizeof(index_str), data->index_type, index); pr_err("index %s is out of range for type %s\n", index_str, @@ -352,9 +292,9 @@ void __ubsan_handle_out_of_bounds(struct out_of_bounds_data *data, void *index) } EXPORT_SYMBOL(__ubsan_handle_out_of_bounds); -void __ubsan_handle_shift_out_of_bounds(struct shift_out_of_bounds_data *data, - void *lhs, void *rhs) +void __ubsan_handle_shift_out_of_bounds(void *_data, void *lhs, void *rhs) { + struct shift_out_of_bounds_data *data = _data; struct type_descriptor *rhs_type = data->rhs_type; struct type_descriptor *lhs_type = data->lhs_type; char rhs_str[VALUE_LENGTH]; @@ -364,7 +304,7 @@ void __ubsan_handle_shift_out_of_bounds(struct shift_out_of_bounds_data *data, if (suppress_report(&data->location)) goto out; - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, "shift-out-of-bounds"); val_to_string(rhs_str, sizeof(rhs_str), rhs_type, rhs); val_to_string(lhs_str, sizeof(lhs_str), lhs_type, lhs); @@ -394,24 +334,25 @@ out: EXPORT_SYMBOL(__ubsan_handle_shift_out_of_bounds); -void __ubsan_handle_builtin_unreachable(struct unreachable_data *data) +void __ubsan_handle_builtin_unreachable(void *_data) { - ubsan_prologue(&data->location); + struct unreachable_data *data = _data; + ubsan_prologue(&data->location, "unreachable"); pr_err("calling __builtin_unreachable()\n"); ubsan_epilogue(); panic("can't return from __builtin_unreachable()"); } EXPORT_SYMBOL(__ubsan_handle_builtin_unreachable); -void __ubsan_handle_load_invalid_value(struct invalid_value_data *data, - void *val) +void __ubsan_handle_load_invalid_value(void *_data, void *val) { + struct invalid_value_data *data = _data; char val_str[VALUE_LENGTH]; if (suppress_report(&data->location)) return; - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, "invalid-load"); val_to_string(val_str, sizeof(val_str), data->type, val); @@ -421,3 +362,34 @@ void __ubsan_handle_load_invalid_value(struct invalid_value_data *data, ubsan_epilogue(); } EXPORT_SYMBOL(__ubsan_handle_load_invalid_value); + +void __ubsan_handle_alignment_assumption(void *_data, unsigned long ptr, + unsigned long align, + unsigned long offset); +void __ubsan_handle_alignment_assumption(void *_data, unsigned long ptr, + unsigned long align, + unsigned long offset) +{ + struct alignment_assumption_data *data = _data; + unsigned long real_ptr; + + if (suppress_report(&data->location)) + return; + + ubsan_prologue(&data->location, "alignment-assumption"); + + if (offset) + pr_err("assumption of %lu byte alignment (with offset of %lu byte) for pointer of type %s failed", + align, offset, data->type->type_name); + else + pr_err("assumption of %lu byte alignment for pointer of type %s failed", + align, data->type->type_name); + + real_ptr = ptr - offset; + pr_err("%saddress is %lu aligned, misalignment offset is %lu bytes", + offset ? "offset " : "", BIT(real_ptr ? __ffs(real_ptr) : 0), + real_ptr & (align - 1)); + + ubsan_epilogue(); +} +EXPORT_SYMBOL(__ubsan_handle_alignment_assumption); diff --git a/lib/ubsan.h b/lib/ubsan.h index 7b56c09473a9..9a0b71c5ff9f 100644 --- a/lib/ubsan.h +++ b/lib/ubsan.h @@ -78,6 +78,12 @@ struct invalid_value_data { struct type_descriptor *type; }; +struct alignment_assumption_data { + struct source_location location; + struct source_location assumption_location; + struct type_descriptor *type; +}; + #if defined(CONFIG_ARCH_SUPPORTS_INT128) typedef __int128 s_max; typedef unsigned __int128 u_max; diff --git a/lib/usercopy.c b/lib/usercopy.c index cbb4d9ec00f2..7413dd300516 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/uaccess.h> #include <linux/bitops.h> +#include <linux/fault-inject-usercopy.h> +#include <linux/instrumented.h> +#include <linux/uaccess.h> /* out-of-line parts */ @@ -9,8 +11,8 @@ unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n { unsigned long res = n; might_fault(); - if (likely(access_ok(from, n))) { - kasan_check_write(to, n); + if (!should_fail_usercopy() && likely(access_ok(from, n))) { + instrument_copy_from_user(to, from, n); res = raw_copy_from_user(to, from, n); } if (unlikely(res)) @@ -24,8 +26,10 @@ EXPORT_SYMBOL(_copy_from_user); unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) { might_fault(); + if (should_fail_usercopy()) + return n; if (likely(access_ok(to, n))) { - kasan_check_read(from, n); + instrument_copy_to_user(to, from, n); n = raw_copy_to_user(to, from, n); } return n; @@ -58,7 +62,7 @@ int check_zeroed_user(const void __user *from, size_t size) from -= align; size += align; - if (!user_access_begin(from, size)) + if (!user_read_access_begin(from, size)) return -EFAULT; unsafe_get_user(val, (unsigned long __user *) from, err_fault); @@ -79,10 +83,10 @@ int check_zeroed_user(const void __user *from, size_t size) val &= aligned_byte_mask(size); done: - user_access_end(); + user_read_access_end(); return (val == 0); err_fault: - user_access_end(); + user_read_access_end(); return -EFAULT; } EXPORT_SYMBOL(check_zeroed_user); diff --git a/lib/uuid.c b/lib/uuid.c index b6a1edb61d87..562d53977cab 100644 --- a/lib/uuid.c +++ b/lib/uuid.c @@ -40,6 +40,16 @@ void generate_random_uuid(unsigned char uuid[16]) } EXPORT_SYMBOL(generate_random_uuid); +void generate_random_guid(unsigned char guid[16]) +{ + get_random_bytes(guid, 16); + /* Set GUID version to 4 --- truly random generation */ + guid[7] = (guid[7] & 0x0F) | 0x40; + /* Set the GUID variant to DCE */ + guid[8] = (guid[8] & 0x3F) | 0x80; +} +EXPORT_SYMBOL(generate_random_guid); + static void __uuid_gen_common(__u8 b[16]) { prandom_bytes(b, 16); diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig index 9fe698ff62ec..d883ac299508 100644 --- a/lib/vdso/Kconfig +++ b/lib/vdso/Kconfig @@ -24,4 +24,10 @@ config GENERIC_COMPAT_VDSO help This config option enables the compat VDSO layer. +config GENERIC_VDSO_TIME_NS + bool + help + Selected by architectures which support time namespaces in the + VDSO + endif diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 9ecfd3b547ba..ce2f69552003 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -2,30 +2,9 @@ /* * Generic userspace implementations of gettimeofday() and similar. */ -#include <linux/compiler.h> -#include <linux/math64.h> -#include <linux/time.h> -#include <linux/kernel.h> -#include <linux/hrtimer_defs.h> #include <vdso/datapage.h> #include <vdso/helpers.h> -/* - * The generic vDSO implementation requires that gettimeofday.h - * provides: - * - __arch_get_vdso_data(): to get the vdso datapage. - * - __arch_get_hw_counter(): to get the hw counter based on the - * clock_mode. - * - gettimeofday_fallback(): fallback for gettimeofday. - * - clock_gettime_fallback(): fallback for clock_gettime. - * - clock_getres_fallback(): fallback for clock_getres. - */ -#ifdef ENABLE_COMPAT_VDSO -#include <asm/vdso/compat_gettimeofday.h> -#else -#include <asm/vdso/gettimeofday.h> -#endif /* ENABLE_COMPAT_VDSO */ - #ifndef vdso_calc_delta /* * Default implementation which works for all sane clocksources. That @@ -38,23 +17,137 @@ u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) } #endif -static int do_hres(const struct vdso_data *vd, clockid_t clk, - struct __kernel_timespec *ts) +#ifndef vdso_shift_ns +static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) { - const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; - u64 cycles, last, sec, ns; + return ns >> shift; +} +#endif + +#ifndef __arch_vdso_hres_capable +static inline bool __arch_vdso_hres_capable(void) +{ + return true; +} +#endif + +#ifndef vdso_clocksource_ok +static inline bool vdso_clocksource_ok(const struct vdso_data *vd) +{ + return vd->clock_mode != VDSO_CLOCKMODE_NONE; +} +#endif + +#ifndef vdso_cycles_ok +static inline bool vdso_cycles_ok(u64 cycles) +{ + return true; +} +#endif + +#ifdef CONFIG_TIME_NS +static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, + struct __kernel_timespec *ts) +{ + const struct vdso_data *vd; + const struct timens_offset *offs = &vdns->offset[clk]; + const struct vdso_timestamp *vdso_ts; + u64 cycles, last, ns; u32 seq; + s64 sec; + + vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE); + vd = __arch_get_timens_vdso_data(vd); + if (clk != CLOCK_MONOTONIC_RAW) + vd = &vd[CS_HRES_COARSE]; + else + vd = &vd[CS_RAW]; + vdso_ts = &vd->basetime[clk]; do { seq = vdso_read_begin(vd); - cycles = __arch_get_hw_counter(vd->clock_mode); + + if (unlikely(!vdso_clocksource_ok(vd))) + return -1; + + cycles = __arch_get_hw_counter(vd->clock_mode, vd); + if (unlikely(!vdso_cycles_ok(cycles))) + return -1; ns = vdso_ts->nsec; last = vd->cycle_last; - if (unlikely((s64)cycles < 0)) + ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); + ns = vdso_shift_ns(ns, vd->shift); + sec = vdso_ts->sec; + } while (unlikely(vdso_read_retry(vd, seq))); + + /* Add the namespace offset */ + sec += offs->sec; + ns += offs->nsec; + + /* + * Do this outside the loop: a race inside the loop could result + * in __iter_div_u64_rem() being extremely slow. + */ + ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); + ts->tv_nsec = ns; + + return 0; +} +#else +static __always_inline +const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) +{ + return NULL; +} + +static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, + struct __kernel_timespec *ts) +{ + return -EINVAL; +} +#endif + +static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, + struct __kernel_timespec *ts) +{ + const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; + u64 cycles, last, sec, ns; + u32 seq; + + /* Allows to compile the high resolution parts out */ + if (!__arch_vdso_hres_capable()) + return -1; + + do { + /* + * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace + * enabled tasks have a special VVAR page installed which + * has vd->seq set to 1 and vd->clock_mode set to + * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks + * this does not affect performance because if vd->seq is + * odd, i.e. a concurrent update is in progress the extra + * check for vd->clock_mode is just a few extra + * instructions while spin waiting for vd->seq to become + * even again. + */ + while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) + return do_hres_timens(vd, clk, ts); + cpu_relax(); + } + smp_rmb(); + + if (unlikely(!vdso_clocksource_ok(vd))) return -1; + cycles = __arch_get_hw_counter(vd->clock_mode, vd); + if (unlikely(!vdso_cycles_ok(cycles))) + return -1; + ns = vdso_ts->nsec; + last = vd->cycle_last; ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); - ns >>= vd->shift; + ns = vdso_shift_ns(ns, vd->shift); sec = vdso_ts->sec; } while (unlikely(vdso_read_retry(vd, seq))); @@ -68,23 +161,73 @@ static int do_hres(const struct vdso_data *vd, clockid_t clk, return 0; } -static void do_coarse(const struct vdso_data *vd, clockid_t clk, - struct __kernel_timespec *ts) +#ifdef CONFIG_TIME_NS +static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, + struct __kernel_timespec *ts) { + const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns); const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; - u32 seq; + const struct timens_offset *offs = &vdns->offset[clk]; + u64 nsec; + s64 sec; + s32 seq; do { seq = vdso_read_begin(vd); + sec = vdso_ts->sec; + nsec = vdso_ts->nsec; + } while (unlikely(vdso_read_retry(vd, seq))); + + /* Add the namespace offset */ + sec += offs->sec; + nsec += offs->nsec; + + /* + * Do this outside the loop: a race inside the loop could result + * in __iter_div_u64_rem() being extremely slow. + */ + ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); + ts->tv_nsec = nsec; + return 0; +} +#else +static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, + struct __kernel_timespec *ts) +{ + return -1; +} +#endif + +static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, + struct __kernel_timespec *ts) +{ + const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; + u32 seq; + + do { + /* + * Open coded to handle VDSO_CLOCK_TIMENS. See comment in + * do_hres(). + */ + while ((seq = READ_ONCE(vd->seq)) & 1) { + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) + return do_coarse_timens(vd, clk, ts); + cpu_relax(); + } + smp_rmb(); + ts->tv_sec = vdso_ts->sec; ts->tv_nsec = vdso_ts->nsec; } while (unlikely(vdso_read_retry(vd, seq))); + + return 0; } -static __maybe_unused int -__cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) +static __always_inline int +__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *ts) { - const struct vdso_data *vd = __arch_get_vdso_data(); u32 msk; /* Check for negative values or invalid clocks */ @@ -96,21 +239,23 @@ __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) * clocks are handled in the VDSO directly. */ msk = 1U << clock; - if (likely(msk & VDSO_HRES)) { - return do_hres(&vd[CS_HRES_COARSE], clock, ts); - } else if (msk & VDSO_COARSE) { - do_coarse(&vd[CS_HRES_COARSE], clock, ts); - return 0; - } else if (msk & VDSO_RAW) { - return do_hres(&vd[CS_RAW], clock, ts); - } - return -1; + if (likely(msk & VDSO_HRES)) + vd = &vd[CS_HRES_COARSE]; + else if (msk & VDSO_COARSE) + return do_coarse(&vd[CS_HRES_COARSE], clock, ts); + else if (msk & VDSO_RAW) + vd = &vd[CS_RAW]; + else + return -1; + + return do_hres(vd, clock, ts); } static __maybe_unused int -__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *ts) { - int ret = __cvdso_clock_gettime_common(clock, ts); + int ret = __cvdso_clock_gettime_common(vd, clock, ts); if (unlikely(ret)) return clock_gettime_fallback(clock, ts); @@ -118,32 +263,42 @@ __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) } static __maybe_unused int -__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) +__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts); +} + +#ifdef BUILD_VDSO32 +static __maybe_unused int +__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, + struct old_timespec32 *res) { struct __kernel_timespec ts; int ret; - ret = __cvdso_clock_gettime_common(clock, &ts); + ret = __cvdso_clock_gettime_common(vd, clock, &ts); -#ifdef VDSO_HAS_32BIT_FALLBACK if (unlikely(ret)) return clock_gettime32_fallback(clock, res); -#else - if (unlikely(ret)) - ret = clock_gettime_fallback(clock, &ts); -#endif - if (likely(!ret)) { - res->tv_sec = ts.tv_sec; - res->tv_nsec = ts.tv_nsec; - } + /* For ret == 0 */ + res->tv_sec = ts.tv_sec; + res->tv_nsec = ts.tv_nsec; + return ret; } static __maybe_unused int -__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) +{ + return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res); +} +#endif /* BUILD_VDSO32 */ + +static __maybe_unused int +__cvdso_gettimeofday_data(const struct vdso_data *vd, + struct __kernel_old_timeval *tv, struct timezone *tz) { - const struct vdso_data *vd = __arch_get_vdso_data(); if (likely(tv != NULL)) { struct __kernel_timespec ts; @@ -156,6 +311,10 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) } if (unlikely(tz != NULL)) { + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) + vd = __arch_get_timens_vdso_data(vd); + tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; } @@ -163,25 +322,41 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) return 0; } +static __maybe_unused int +__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz); +} + #ifdef VDSO_HAS_TIME -static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) +static __maybe_unused __kernel_old_time_t +__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) { - const struct vdso_data *vd = __arch_get_vdso_data(); - __kernel_old_time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); + __kernel_old_time_t t; + + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) + vd = __arch_get_timens_vdso_data(vd); + + t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); if (time) *time = t; return t; } + +static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) +{ + return __cvdso_time_data(__arch_get_vdso_data(), time); +} #endif /* VDSO_HAS_TIME */ #ifdef VDSO_HAS_CLOCK_GETRES static __maybe_unused -int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) +int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *res) { - const struct vdso_data *vd = __arch_get_vdso_data(); - u64 hrtimer_res; u32 msk; u64 ns; @@ -189,27 +364,25 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) if (unlikely((u32) clock >= MAX_CLOCKS)) return -1; - hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) + vd = __arch_get_timens_vdso_data(vd); + /* * Convert the clockid to a bitmask and use it to check which * clocks are handled in the VDSO directly. */ msk = 1U << clock; - if (msk & VDSO_HRES) { + if (msk & (VDSO_HRES | VDSO_RAW)) { /* * Preserves the behaviour of posix_get_hrtimer_res(). */ - ns = hrtimer_res; + ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); } else if (msk & VDSO_COARSE) { /* * Preserves the behaviour of posix_get_coarse_res(). */ ns = LOW_RES_NSEC; - } else if (msk & VDSO_RAW) { - /* - * Preserves the behaviour of posix_get_hrtimer_res(). - */ - ns = hrtimer_res; } else { return -1; } @@ -221,35 +394,48 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) return 0; } -int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) +static __maybe_unused +int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *res) { - int ret = __cvdso_clock_getres_common(clock, res); + int ret = __cvdso_clock_getres_common(vd, clock, res); if (unlikely(ret)) return clock_getres_fallback(clock, res); return 0; } +static __maybe_unused +int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) +{ + return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res); +} + +#ifdef BUILD_VDSO32 static __maybe_unused int -__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) +__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, + struct old_timespec32 *res) { struct __kernel_timespec ts; int ret; - ret = __cvdso_clock_getres_common(clock, &ts); + ret = __cvdso_clock_getres_common(vd, clock, &ts); -#ifdef VDSO_HAS_32BIT_FALLBACK if (unlikely(ret)) return clock_getres32_fallback(clock, res); -#else - if (unlikely(ret)) - ret = clock_getres_fallback(clock, &ts); -#endif - if (likely(!ret && res)) { + if (likely(res)) { res->tv_sec = ts.tv_sec; res->tv_nsec = ts.tv_nsec; } return ret; } + +static __maybe_unused int +__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) +{ + return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(), + clock, res); +} +#endif /* BUILD_VDSO32 */ #endif /* VDSO_HAS_CLOCK_GETRES */ diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 7c488a1ce318..58d5e567f836 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -17,7 +17,7 @@ * - scnprintf and vscnprintf */ -#include <stdarg.h> +#include <linux/stdarg.h> #include <linux/build_bug.h> #include <linux/clk.h> #include <linux/clk-provider.h> @@ -34,6 +34,7 @@ #include <linux/dcache.h> #include <linux/cred.h> #include <linux/rtc.h> +#include <linux/time.h> #include <linux/uuid.h> #include <linux/of.h> #include <net/addrconf.h> @@ -52,28 +53,42 @@ #include <linux/string_helpers.h> #include "kstrtox.h" +static noinline unsigned long long simple_strntoull(const char *startp, size_t max_chars, char **endp, unsigned int base) +{ + const char *cp; + unsigned long long result = 0ULL; + size_t prefix_chars; + unsigned int rv; + + cp = _parse_integer_fixup_radix(startp, &base); + prefix_chars = cp - startp; + if (prefix_chars < max_chars) { + rv = _parse_integer_limit(cp, base, &result, max_chars - prefix_chars); + /* FIXME */ + cp += (rv & ~KSTRTOX_OVERFLOW); + } else { + /* Field too short for prefix + digit, skip over without converting */ + cp = startp + max_chars; + } + + if (endp) + *endp = (char *)cp; + + return result; +} + /** * simple_strtoull - convert a string to an unsigned long long * @cp: The start of the string * @endp: A pointer to the end of the parsed string will be placed here * @base: The number base to use * - * This function is obsolete. Please use kstrtoull instead. + * This function has caveats. Please use kstrtoull instead. */ +noinline unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base) { - unsigned long long result; - unsigned int rv; - - cp = _parse_integer_fixup_radix(cp, &base); - rv = _parse_integer(cp, base, &result); - /* FIXME */ - cp += (rv & ~KSTRTOX_OVERFLOW); - - if (endp) - *endp = (char *)cp; - - return result; + return simple_strntoull(cp, INT_MAX, endp, base); } EXPORT_SYMBOL(simple_strtoull); @@ -83,7 +98,7 @@ EXPORT_SYMBOL(simple_strtoull); * @endp: A pointer to the end of the parsed string will be placed here * @base: The number base to use * - * This function is obsolete. Please use kstrtoul instead. + * This function has caveats. Please use kstrtoul instead. */ unsigned long simple_strtoul(const char *cp, char **endp, unsigned int base) { @@ -97,7 +112,7 @@ EXPORT_SYMBOL(simple_strtoul); * @endp: A pointer to the end of the parsed string will be placed here * @base: The number base to use * - * This function is obsolete. Please use kstrtol instead. + * This function has caveats. Please use kstrtol instead. */ long simple_strtol(const char *cp, char **endp, unsigned int base) { @@ -108,20 +123,32 @@ long simple_strtol(const char *cp, char **endp, unsigned int base) } EXPORT_SYMBOL(simple_strtol); +static long long simple_strntoll(const char *cp, size_t max_chars, char **endp, + unsigned int base) +{ + /* + * simple_strntoull() safely handles receiving max_chars==0 in the + * case cp[0] == '-' && max_chars == 1. + * If max_chars == 0 we can drop through and pass it to simple_strntoull() + * and the content of *cp is irrelevant. + */ + if (*cp == '-' && max_chars > 0) + return -simple_strntoull(cp + 1, max_chars - 1, endp, base); + + return simple_strntoull(cp, max_chars, endp, base); +} + /** * simple_strtoll - convert a string to a signed long long * @cp: The start of the string * @endp: A pointer to the end of the parsed string will be placed here * @base: The number base to use * - * This function is obsolete. Please use kstrtoll instead. + * This function has caveats. Please use kstrtoll instead. */ long long simple_strtoll(const char *cp, char **endp, unsigned int base) { - if (*cp == '-') - return -simple_strtoull(cp + 1, endp, base); - - return simple_strtoull(cp, endp, base); + return simple_strntoll(cp, INT_MAX, endp, base); } EXPORT_SYMBOL(simple_strtoll); @@ -380,6 +407,10 @@ int num_to_str(char *buf, int size, unsigned long long num, unsigned int width) #define SMALL 32 /* use lowercase in hex (must be 32 == 0x20) */ #define SPECIAL 64 /* prefix hex with "0x", octal with "0" */ +static_assert(SIGN == 1); +static_assert(ZEROPAD == ('0' - ' ')); +static_assert(SMALL == ('a' ^ 'A')); + enum format_type { FORMAT_TYPE_NONE, /* Just a string part */ FORMAT_TYPE_WIDTH, @@ -506,7 +537,7 @@ char *number(char *buf, char *end, unsigned long long num, /* zero or space padding */ if (!(spec.flags & LEFT)) { char c = ' ' + (spec.flags & ZEROPAD); - BUILD_BUG_ON(' ' + ZEROPAD != '0'); + while (--field_width >= 0) { if (buf < end) *buf = c; @@ -794,6 +825,13 @@ static char *ptr_to_id(char *buf, char *end, const void *ptr, unsigned long hashval; int ret; + /* + * Print the real pointer value for NULL and error pointers, + * as they are not actual addresses. + */ + if (IS_ERR_OR_NULL(ptr)) + return pointer_string(buf, end, ptr, spec); + /* When debugging early boot use non-cryptographically secure hash. */ if (unlikely(debug_boot_weak_hash)) { hashval = hash_long((unsigned long)ptr, 32); @@ -929,13 +967,13 @@ char *bdev_name(char *buf, char *end, struct block_device *bdev, hd = bdev->bd_disk; buf = string(buf, end, hd->disk_name, spec); - if (bdev->bd_part->partno) { + if (bdev->bd_partno) { if (isdigit(hd->disk_name[strlen(hd->disk_name)-1])) { if (buf < end) *buf = 'p'; buf++; } - buf = number(buf, end, bdev->bd_part->partno, spec); + buf = number(buf, end, bdev->bd_partno, spec); } return buf; } @@ -955,8 +993,12 @@ char *symbol_string(char *buf, char *end, void *ptr, value = (unsigned long)ptr; #ifdef CONFIG_KALLSYMS - if (*fmt == 'B') + if (*fmt == 'B' && fmt[1] == 'b') + sprint_backtrace_build_id(sym, value); + else if (*fmt == 'B') sprint_backtrace(sym, value); + else if (*fmt == 'S' && (fmt[1] == 'b' || (fmt[1] == 'R' && fmt[2] == 'b'))) + sprint_symbol_build_id(sym, value); else if (*fmt != 's') sprint_symbol(sym, value); else @@ -1254,7 +1296,7 @@ char *mac_address_string(char *buf, char *end, u8 *addr, case 'R': reversed = true; - /* fall through */ + fallthrough; default: separator = ':'; @@ -1670,7 +1712,8 @@ char *uuid_string(char *buf, char *end, const u8 *addr, switch (*(++fmt)) { case 'L': - uc = true; /* fall-through */ + uc = true; + fallthrough; case 'l': index = guid_index; break; @@ -1722,6 +1765,42 @@ char *netdev_bits(char *buf, char *end, const void *addr, } static noinline_for_stack +char *fourcc_string(char *buf, char *end, const u32 *fourcc, + struct printf_spec spec, const char *fmt) +{ + char output[sizeof("0123 little-endian (0x01234567)")]; + char *p = output; + unsigned int i; + u32 val; + + if (fmt[1] != 'c' || fmt[2] != 'c') + return error_string(buf, end, "(%p4?)", spec); + + if (check_pointer(&buf, end, fourcc, spec)) + return buf; + + val = *fourcc & ~BIT(31); + + for (i = 0; i < sizeof(*fourcc); i++) { + unsigned char c = val >> (i * 8); + + /* Print non-control ASCII characters as-is, dot otherwise */ + *p++ = isascii(c) && isprint(c) ? c : '.'; + } + + strcpy(p, *fourcc & BIT(31) ? " big-endian" : " little-endian"); + p += strlen(p); + + *p++ = ' '; + *p++ = '('; + p = special_hex_number(p, output + sizeof(output) - 2, *fourcc, sizeof(u32)); + *p++ = ')'; + *p = '\0'; + + return string(buf, end, output, spec); +} + +static noinline_for_stack char *address_val(char *buf, char *end, const void *addr, struct printf_spec spec, const char *fmt) { @@ -1786,7 +1865,8 @@ char *rtc_str(char *buf, char *end, const struct rtc_time *tm, struct printf_spec spec, const char *fmt) { bool have_t = true, have_d = true; - bool raw = false; + bool raw = false, iso8601_separator = true; + bool found = true; int count = 2; if (check_pointer(&buf, end, tm, spec)) @@ -1803,14 +1883,25 @@ char *rtc_str(char *buf, char *end, const struct rtc_time *tm, break; } - raw = fmt[count] == 'r'; + do { + switch (fmt[count++]) { + case 'r': + raw = true; + break; + case 's': + iso8601_separator = false; + break; + default: + found = false; + break; + } + } while (found); if (have_d) buf = date_str(buf, end, tm, raw); if (have_d && have_t) { - /* Respect ISO 8601 */ if (buf < end) - *buf = 'T'; + *buf = iso8601_separator ? 'T' : ' '; buf++; } if (have_t) @@ -1820,14 +1911,39 @@ char *rtc_str(char *buf, char *end, const struct rtc_time *tm, } static noinline_for_stack +char *time64_str(char *buf, char *end, const time64_t time, + struct printf_spec spec, const char *fmt) +{ + struct rtc_time rtc_time; + struct tm tm; + + time64_to_tm(time, 0, &tm); + + rtc_time.tm_sec = tm.tm_sec; + rtc_time.tm_min = tm.tm_min; + rtc_time.tm_hour = tm.tm_hour; + rtc_time.tm_mday = tm.tm_mday; + rtc_time.tm_mon = tm.tm_mon; + rtc_time.tm_year = tm.tm_year; + rtc_time.tm_wday = tm.tm_wday; + rtc_time.tm_yday = tm.tm_yday; + + rtc_time.tm_isdst = 0; + + return rtc_str(buf, end, &rtc_time, spec, fmt); +} + +static noinline_for_stack char *time_and_date(char *buf, char *end, void *ptr, struct printf_spec spec, const char *fmt) { switch (fmt[1]) { case 'R': return rtc_str(buf, end, (const struct rtc_time *)ptr, spec, fmt); + case 'T': + return time64_str(buf, end, *(const time64_t *)ptr, spec, fmt); default: - return error_string(buf, end, "(%ptR?)", spec); + return error_string(buf, end, "(%pt?)", spec); } } @@ -1879,6 +1995,74 @@ char *format_flags(char *buf, char *end, unsigned long flags, return buf; } +struct page_flags_fields { + int width; + int shift; + int mask; + const struct printf_spec *spec; + const char *name; +}; + +static const struct page_flags_fields pff[] = { + {SECTIONS_WIDTH, SECTIONS_PGSHIFT, SECTIONS_MASK, + &default_dec_spec, "section"}, + {NODES_WIDTH, NODES_PGSHIFT, NODES_MASK, + &default_dec_spec, "node"}, + {ZONES_WIDTH, ZONES_PGSHIFT, ZONES_MASK, + &default_dec_spec, "zone"}, + {LAST_CPUPID_WIDTH, LAST_CPUPID_PGSHIFT, LAST_CPUPID_MASK, + &default_flag_spec, "lastcpupid"}, + {KASAN_TAG_WIDTH, KASAN_TAG_PGSHIFT, KASAN_TAG_MASK, + &default_flag_spec, "kasantag"}, +}; + +static +char *format_page_flags(char *buf, char *end, unsigned long flags) +{ + unsigned long main_flags = flags & PAGEFLAGS_MASK; + bool append = false; + int i; + + buf = number(buf, end, flags, default_flag_spec); + if (buf < end) + *buf = '('; + buf++; + + /* Page flags from the main area. */ + if (main_flags) { + buf = format_flags(buf, end, main_flags, pageflag_names); + append = true; + } + + /* Page flags from the fields area */ + for (i = 0; i < ARRAY_SIZE(pff); i++) { + /* Skip undefined fields. */ + if (!pff[i].width) + continue; + + /* Format: Flag Name + '=' (equals sign) + Number + '|' (separator) */ + if (append) { + if (buf < end) + *buf = '|'; + buf++; + } + + buf = string(buf, end, pff[i].name, default_str_spec); + if (buf < end) + *buf = '='; + buf++; + buf = number(buf, end, (flags >> pff[i].shift) & pff[i].mask, + *pff[i].spec); + + append = true; + } + if (buf < end) + *buf = ')'; + buf++; + + return buf; +} + static noinline_for_stack char *flags_string(char *buf, char *end, void *flags_ptr, struct printf_spec spec, const char *fmt) @@ -1891,17 +2075,13 @@ char *flags_string(char *buf, char *end, void *flags_ptr, switch (fmt[1]) { case 'p': - flags = *(unsigned long *)flags_ptr; - /* Remove zone id */ - flags &= (1UL << NR_PAGEFLAGS) - 1; - names = pageflag_names; - break; + return format_page_flags(buf, end, *(unsigned long *)flags_ptr); case 'v': flags = *(unsigned long *)flags_ptr; names = vmaflag_names; break; case 'g': - flags = *(gfp_t *)flags_ptr; + flags = (__force unsigned long)(*(gfp_t *)flags_ptr); names = gfpflag_names; break; default: @@ -1943,12 +2123,6 @@ char *device_node_string(char *buf, char *end, struct device_node *dn, char *buf_start = buf; struct property *prop; bool has_mult, pass; - static const struct printf_spec num_spec = { - .flags = SMALL, - .field_width = -1, - .precision = -1, - .base = 10, - }; struct printf_spec str_spec = spec; str_spec.field_width = -1; @@ -1988,7 +2162,7 @@ char *device_node_string(char *buf, char *end, struct device_node *dn, str_spec.precision = precision; break; case 'p': /* phandle */ - buf = number(buf, end, (unsigned int)dn->phandle, num_spec); + buf = number(buf, end, (unsigned int)dn->phandle, default_dec_spec); break; case 'P': /* path-spec */ p = fwnode_get_name(of_fwnode_handle(dn)); @@ -2059,6 +2233,35 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode, return widen_string(buf, buf - buf_start, end, spec); } +/* Disable pointer hashing if requested */ +bool no_hash_pointers __ro_after_init; +EXPORT_SYMBOL_GPL(no_hash_pointers); + +int __init no_hash_pointers_enable(char *str) +{ + if (no_hash_pointers) + return 0; + + no_hash_pointers = true; + + pr_warn("**********************************************************\n"); + pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); + pr_warn("** **\n"); + pr_warn("** This system shows unhashed kernel memory addresses **\n"); + pr_warn("** via the console, logs, and other interfaces. This **\n"); + pr_warn("** might reduce the security of your system. **\n"); + pr_warn("** **\n"); + pr_warn("** If you see this message and you are not debugging **\n"); + pr_warn("** the kernel, report this immediately to your system **\n"); + pr_warn("** administrator! **\n"); + pr_warn("** **\n"); + pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); + pr_warn("**********************************************************\n"); + + return 0; +} +early_param("no_hash_pointers", no_hash_pointers_enable); + /* * Show a '%p' thing. A kernel extension is that the '%p' is followed * by an extra set of alphanumeric characters that are extended format @@ -2072,9 +2275,11 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode, * - 'S' For symbolic direct pointers (or function descriptors) with offset * - 's' For symbolic direct pointers (or function descriptors) without offset * - '[Ss]R' as above with __builtin_extract_return_addr() translation + * - 'S[R]b' as above with module build ID (for use in backtraces) * - '[Ff]' %pf and %pF were obsoleted and later removed in favor of * %ps and %pS. Be careful when re-using these specifiers. * - 'B' For backtraced symbolic direct pointers with offset + * - 'Bb' as above with module build ID (for use in backtraces) * - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref] * - 'r' For raw struct resource, e.g., [mem 0x0-0x1f flags 0x201] * - 'b[l]' For a bitmap, the number of bits is determined by the field @@ -2101,7 +2306,7 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode, * [4] or [6] and is able to print port [p], flowinfo [f], scope [s] * - '[Ii][4S][hnbl]' IPv4 addresses in host, network, big or little endian order * - 'I[6S]c' for IPv6 addresses printed as specified by - * http://tools.ietf.org/html/rfc5952 + * https://tools.ietf.org/html/rfc5952 * - 'E[achnops]' For an escaped buffer, where rules are defined by combination * of the following flags (see string_escape_mem() for the * details): @@ -2129,8 +2334,11 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode, * Implements a "recursive vsnprintf". * Do not use this feature without some mechanism to verify the * correctness of the format string and va_list arguments. - * - 'K' For a kernel pointer that should be hidden from unprivileged users + * - 'K' For a kernel pointer that should be hidden from unprivileged users. + * Use only for procfs, sysfs and similar files, not printk(); please + * read the documentation (path below) first. * - 'NF' For a netdev_features_t + * - '4cc' V4L2 or DRM FourCC code, with endianness and raw numerical value. * - 'h[CDN]' For a variable-length buffer, it prints it as a hex string with * a certain separator (' ' by default): * C colon @@ -2143,8 +2351,9 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode, * - 'd[234]' For a dentry name (optionally 2-4 last components) * - 'D[234]' Same as 'd' but for a struct file * - 'g' For block_device name (gendisk + partition number) - * - 't[R][dt][r]' For time and date as represented: + * - 't[RT][dt][r][s]' For time and date as represented by: * R struct rtc_time + * T time64_t * - 'C' For a clock, it prints the name (Common Clock Framework) or address * (legacy clock framework) of the clock * - 'Cn' For a clock, it prints the name (Common Clock Framework) or address @@ -2167,7 +2376,12 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode, * Without an option prints the full name of the node * f full name * P node name, including a possible unit address - * - 'x' For printing the address. Equivalent to "%lx". + * - 'x' For printing the address unmodified. Equivalent to "%lx". + * Please read the documentation (path below) before using! + * - '[ku]s' For a BPF/tracing related format specifier, e.g. used out of + * bpf_trace_printk() where [ku] prefix specifies either kernel (k) + * or user (u) memory to probe, and: + * s a string, equivalent to "%s" on direct vsnprintf() use * * ** When making changes please also update: * Documentation/core-api/printk-formats.rst @@ -2183,7 +2397,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, case 'S': case 's': ptr = dereference_symbol_descriptor(ptr); - /* Fallthrough */ + fallthrough; case 'B': return symbol_string(buf, end, ptr, spec, fmt); case 'R': @@ -2223,6 +2437,8 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, return restricted_pointer(buf, end, ptr, spec); case 'N': return netdev_bits(buf, end, ptr, spec, fmt); + case '4': + return fourcc_string(buf, end, ptr, spec, fmt); case 'a': return address_val(buf, end, ptr, spec, fmt); case 'd': @@ -2251,10 +2467,24 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, if (!IS_ERR(ptr)) break; return err_ptr(buf, end, ptr, spec); + case 'u': + case 'k': + switch (fmt[1]) { + case 's': + return string(buf, end, ptr, spec); + default: + return error_string(buf, end, "(einval)", spec); + } } - /* default is to _not_ leak addresses, hash before printing */ - return ptr_to_id(buf, end, ptr, spec); + /* + * default is to _not_ leak addresses, so hash before printing, + * unless no_hash_pointers is specified on the command line. + */ + if (unlikely(no_hash_pointers)) + return pointer_string(buf, end, ptr, spec); + else + return ptr_to_id(buf, end, ptr, spec); } /* @@ -2406,7 +2636,7 @@ qualifier: case 'x': spec->flags |= SMALL; - /* fall through */ + fallthrough; case 'X': spec->base = 16; @@ -2415,6 +2645,7 @@ qualifier: case 'd': case 'i': spec->flags |= SIGN; + break; case 'u': break; @@ -2424,7 +2655,7 @@ qualifier: * utility, treat it as any other invalid or * unsupported format specifier. */ - /* Fall-through */ + fallthrough; default: WARN_ONCE(1, "Please remove unsupported %%%c in format string\n", *fmt); @@ -3058,8 +3289,6 @@ int bstr_printf(char *buf, size_t size, const char *fmt, const u32 *bin_buf) switch (*fmt) { case 'S': case 's': - case 'F': - case 'f': case 'x': case 'K': case 'e': @@ -3202,7 +3431,7 @@ int vsscanf(const char *buf, const char *fmt, va_list args) while (*fmt) { /* skip any white space in format */ - /* white space in format matchs any amount of + /* white space in format matches any amount of * white space, including none, in the input. */ if (isspace(*fmt)) { @@ -3367,10 +3596,10 @@ int vsscanf(const char *buf, const char *fmt, va_list args) break; case 'i': base = 0; - /* fall through */ + fallthrough; case 'd': is_sign = true; - /* fall through */ + fallthrough; case 'u': break; case '%': @@ -3389,8 +3618,12 @@ int vsscanf(const char *buf, const char *fmt, va_list args) str = skip_spaces(str); digit = *str; - if (is_sign && digit == '-') + if (is_sign && digit == '-') { + if (field_width == 1) + break; + digit = *(str + 1); + } if (!digit || (base == 16 && !isxdigit(digit)) @@ -3400,25 +3633,13 @@ int vsscanf(const char *buf, const char *fmt, va_list args) break; if (is_sign) - val.s = qualifier != 'L' ? - simple_strtol(str, &next, base) : - simple_strtoll(str, &next, base); + val.s = simple_strntoll(str, + field_width >= 0 ? field_width : INT_MAX, + &next, base); else - val.u = qualifier != 'L' ? - simple_strtoul(str, &next, base) : - simple_strtoull(str, &next, base); - - if (field_width > 0 && next - str > field_width) { - if (base == 0) - _parse_integer_fixup_radix(str, &base); - while (next - str > field_width) { - if (is_sign) - val.s = div_s64(val.s, base); - else - val.u = div_u64(val.u, base); - --next; - } - } + val.u = simple_strntoull(str, + field_width >= 0 ? field_width : INT_MAX, + &next, base); switch (qualifier) { case 'H': /* that's 'hh' in format */ diff --git a/lib/xarray.c b/lib/xarray.c index 1237c213f52b..f5d8f54907b4 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0+ /* * XArray implementation - * Copyright (c) 2017 Microsoft Corporation + * Copyright (c) 2017-2018 Microsoft Corporation + * Copyright (c) 2018-2020 Oracle * Author: Matthew Wilcox <willy@infradead.org> */ @@ -265,13 +266,14 @@ static void xa_node_free(struct xa_node *node) */ static void xas_destroy(struct xa_state *xas) { - struct xa_node *node = xas->xa_alloc; + struct xa_node *next, *node = xas->xa_alloc; - if (!node) - return; - XA_NODE_BUG_ON(node, !list_empty(&node->private_list)); - kmem_cache_free(radix_tree_node_cachep, node); - xas->xa_alloc = NULL; + while (node) { + XA_NODE_BUG_ON(node, !list_empty(&node->private_list)); + next = rcu_dereference_raw(node->parent); + radix_tree_node_rcu_free(&node->rcu_head); + xas->xa_alloc = node = next; + } } /** @@ -303,6 +305,7 @@ bool xas_nomem(struct xa_state *xas, gfp_t gfp) xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp); if (!xas->xa_alloc) return false; + xas->xa_alloc->parent = NULL; XA_NODE_BUG_ON(xas->xa_alloc, !list_empty(&xas->xa_alloc->private_list)); xas->xa_node = XAS_RESTART; return true; @@ -338,6 +341,7 @@ static bool __xas_nomem(struct xa_state *xas, gfp_t gfp) } if (!xas->xa_alloc) return false; + xas->xa_alloc->parent = NULL; XA_NODE_BUG_ON(xas->xa_alloc, !list_empty(&xas->xa_alloc->private_list)); xas->xa_node = XAS_RESTART; return true; @@ -402,7 +406,7 @@ static unsigned long xas_size(const struct xa_state *xas) /* * Use this to calculate the maximum index that will need to be created * in order to add the entry described by @xas. Because we cannot store a - * multiple-index entry at index 0, the calculation is a little more complex + * multi-index entry at index 0, the calculation is a little more complex * than you might expect. */ static unsigned long xas_max(struct xa_state *xas) @@ -702,7 +706,7 @@ void xas_create_range(struct xa_state *xas) unsigned char shift = xas->xa_shift; unsigned char sibs = xas->xa_sibs; - xas->xa_index |= ((sibs + 1) << shift) - 1; + xas->xa_index |= ((sibs + 1UL) << shift) - 1; if (xas_is_node(xas) && xas->xa_node->shift == xas->xa_shift) xas->xa_offset |= sibs; xas->xa_shift = 0; @@ -945,6 +949,154 @@ void xas_init_marks(const struct xa_state *xas) } EXPORT_SYMBOL_GPL(xas_init_marks); +#ifdef CONFIG_XARRAY_MULTI +static unsigned int node_get_marks(struct xa_node *node, unsigned int offset) +{ + unsigned int marks = 0; + xa_mark_t mark = XA_MARK_0; + + for (;;) { + if (node_get_mark(node, offset, mark)) + marks |= 1 << (__force unsigned int)mark; + if (mark == XA_MARK_MAX) + break; + mark_inc(mark); + } + + return marks; +} + +static void node_set_marks(struct xa_node *node, unsigned int offset, + struct xa_node *child, unsigned int marks) +{ + xa_mark_t mark = XA_MARK_0; + + for (;;) { + if (marks & (1 << (__force unsigned int)mark)) { + node_set_mark(node, offset, mark); + if (child) + node_mark_all(child, mark); + } + if (mark == XA_MARK_MAX) + break; + mark_inc(mark); + } +} + +/** + * xas_split_alloc() - Allocate memory for splitting an entry. + * @xas: XArray operation state. + * @entry: New entry which will be stored in the array. + * @order: Current entry order. + * @gfp: Memory allocation flags. + * + * This function should be called before calling xas_split(). + * If necessary, it will allocate new nodes (and fill them with @entry) + * to prepare for the upcoming split of an entry of @order size into + * entries of the order stored in the @xas. + * + * Context: May sleep if @gfp flags permit. + */ +void xas_split_alloc(struct xa_state *xas, void *entry, unsigned int order, + gfp_t gfp) +{ + unsigned int sibs = (1 << (order % XA_CHUNK_SHIFT)) - 1; + unsigned int mask = xas->xa_sibs; + + /* XXX: no support for splitting really large entries yet */ + if (WARN_ON(xas->xa_shift + 2 * XA_CHUNK_SHIFT < order)) + goto nomem; + if (xas->xa_shift + XA_CHUNK_SHIFT > order) + return; + + do { + unsigned int i; + void *sibling = NULL; + struct xa_node *node; + + node = kmem_cache_alloc(radix_tree_node_cachep, gfp); + if (!node) + goto nomem; + node->array = xas->xa; + for (i = 0; i < XA_CHUNK_SIZE; i++) { + if ((i & mask) == 0) { + RCU_INIT_POINTER(node->slots[i], entry); + sibling = xa_mk_sibling(i); + } else { + RCU_INIT_POINTER(node->slots[i], sibling); + } + } + RCU_INIT_POINTER(node->parent, xas->xa_alloc); + xas->xa_alloc = node; + } while (sibs-- > 0); + + return; +nomem: + xas_destroy(xas); + xas_set_err(xas, -ENOMEM); +} +EXPORT_SYMBOL_GPL(xas_split_alloc); + +/** + * xas_split() - Split a multi-index entry into smaller entries. + * @xas: XArray operation state. + * @entry: New entry to store in the array. + * @order: Current entry order. + * + * The size of the new entries is set in @xas. The value in @entry is + * copied to all the replacement entries. + * + * Context: Any context. The caller should hold the xa_lock. + */ +void xas_split(struct xa_state *xas, void *entry, unsigned int order) +{ + unsigned int sibs = (1 << (order % XA_CHUNK_SHIFT)) - 1; + unsigned int offset, marks; + struct xa_node *node; + void *curr = xas_load(xas); + int values = 0; + + node = xas->xa_node; + if (xas_top(node)) + return; + + marks = node_get_marks(node, xas->xa_offset); + + offset = xas->xa_offset + sibs; + do { + if (xas->xa_shift < node->shift) { + struct xa_node *child = xas->xa_alloc; + + xas->xa_alloc = rcu_dereference_raw(child->parent); + child->shift = node->shift - XA_CHUNK_SHIFT; + child->offset = offset; + child->count = XA_CHUNK_SIZE; + child->nr_values = xa_is_value(entry) ? + XA_CHUNK_SIZE : 0; + RCU_INIT_POINTER(child->parent, node); + node_set_marks(node, offset, child, marks); + rcu_assign_pointer(node->slots[offset], + xa_mk_node(child)); + if (xa_is_value(curr)) + values--; + } else { + unsigned int canon = offset - xas->xa_sibs; + + node_set_marks(node, canon, NULL, marks); + rcu_assign_pointer(node->slots[canon], entry); + while (offset > canon) + rcu_assign_pointer(node->slots[offset--], + xa_mk_sibling(canon)); + values += (xa_is_value(entry) - xa_is_value(curr)) * + (xas->xa_sibs + 1); + } + } while (offset-- > xas->xa_offset); + + node->nr_values += values; +} +EXPORT_SYMBOL_GPL(xas_split); +#endif + /** * xas_pause() - Pause a walk to drop a lock. * @xas: XArray operation state. @@ -967,17 +1119,19 @@ void xas_pause(struct xa_state *xas) if (xas_invalid(xas)) return; + xas->xa_node = XAS_RESTART; if (node) { - unsigned int offset = xas->xa_offset; + unsigned long offset = xas->xa_offset; while (++offset < XA_CHUNK_SIZE) { if (!xa_is_sibling(xa_entry(xas->xa, node, offset))) break; } xas->xa_index += (offset - xas->xa_offset) << node->shift; + if (xas->xa_index == 0) + xas->xa_node = XAS_BOUNDS; } else { xas->xa_index++; } - xas->xa_node = XAS_RESTART; } EXPORT_SYMBOL_GPL(xas_pause); @@ -1079,13 +1233,15 @@ void *xas_find(struct xa_state *xas, unsigned long max) { void *entry; - if (xas_error(xas)) + if (xas_error(xas) || xas->xa_node == XAS_BOUNDS) return NULL; + if (xas->xa_index > max) + return set_bounds(xas); if (!xas->xa_node) { xas->xa_index = 1; return set_bounds(xas); - } else if (xas_top(xas->xa_node)) { + } else if (xas->xa_node == XAS_RESTART) { entry = xas_load(xas); if (entry || xas_not_node(xas->xa_node)) return entry; @@ -1150,6 +1306,8 @@ void *xas_find_marked(struct xa_state *xas, unsigned long max, xa_mark_t mark) if (xas_error(xas)) return NULL; + if (xas->xa_index > max) + goto max; if (!xas->xa_node) { xas->xa_index = 1; @@ -1201,6 +1359,8 @@ void *xas_find_marked(struct xa_state *xas, unsigned long max, xa_mark_t mark) } entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset); + if (!entry && !(xa_track_free(xas->xa) && mark == XA_FREE_MARK)) + continue; if (!xa_is_node(entry)) return entry; xas->xa_node = xa_to_node(entry); @@ -1398,7 +1558,7 @@ EXPORT_SYMBOL(__xa_store); * @gfp: Memory allocation flags. * * After this function returns, loads from this index will return @entry. - * Storing into an existing multislot entry updates the entry of every index. + * Storing into an existing multi-index entry updates the entry of every index. * The marks associated with @index are unaffected unless @entry is %NULL. * * Context: Any context. Takes and releases the xa_lock. @@ -1540,7 +1700,7 @@ static void xas_set_range(struct xa_state *xas, unsigned long first, * * After this function returns, loads from any index between @first and @last, * inclusive will return @entry. - * Storing into an existing multislot entry updates the entry of every index. + * Storing into an existing multi-index entry updates the entry of every index. * The marks associated with @index are unaffected unless @entry is %NULL. * * Context: Process context. Takes and releases the xa_lock. May sleep @@ -1583,6 +1743,46 @@ unlock: return xas_result(&xas, NULL); } EXPORT_SYMBOL(xa_store_range); + +/** + * xa_get_order() - Get the order of an entry. + * @xa: XArray. + * @index: Index of the entry. + * + * Return: A number between 0 and 63 indicating the order of the entry. + */ +int xa_get_order(struct xarray *xa, unsigned long index) +{ + XA_STATE(xas, xa, index); + void *entry; + int order = 0; + + rcu_read_lock(); + entry = xas_load(&xas); + + if (!entry) + goto unlock; + + if (!xas.xa_node) + goto unlock; + + for (;;) { + unsigned int slot = xas.xa_offset + (1 << order); + + if (slot >= XA_CHUNK_SIZE) + break; + if (!xa_is_sibling(xas.xa_node->slots[slot])) + break; + order++; + } + + order += xas.xa_node->shift; +unlock: + rcu_read_unlock(); + + return order; +} +EXPORT_SYMBOL(xa_get_order); #endif /* CONFIG_XARRAY_MULTI */ /** @@ -1824,6 +2024,18 @@ void *xa_find(struct xarray *xa, unsigned long *indexp, } EXPORT_SYMBOL(xa_find); +static bool xas_sibling(struct xa_state *xas) +{ + struct xa_node *node = xas->xa_node; + unsigned long mask; + + if (!IS_ENABLED(CONFIG_XARRAY_MULTI) || !node) + return false; + mask = (XA_CHUNK_SIZE << node->shift) - 1; + return (xas->xa_index & mask) > + ((unsigned long)xas->xa_offset << node->shift); +} + /** * xa_find_after() - Search the XArray for a present entry. * @xa: XArray. @@ -1847,21 +2059,20 @@ void *xa_find_after(struct xarray *xa, unsigned long *indexp, XA_STATE(xas, xa, *indexp + 1); void *entry; + if (xas.xa_index == 0) + return NULL; + rcu_read_lock(); for (;;) { if ((__force unsigned int)filter < XA_MAX_MARKS) entry = xas_find_marked(&xas, max, filter); else entry = xas_find(&xas, max); - if (xas.xa_node == XAS_BOUNDS) + + if (xas_invalid(&xas)) break; - if (xas.xa_shift) { - if (xas.xa_index & ((1UL << xas.xa_shift) - 1)) - continue; - } else { - if (xas.xa_offset < (xas.xa_index & XA_CHUNK_MASK)) - continue; - } + if (xas_sibling(&xas)) + continue; if (!xas_retry(&xas, entry)) break; } @@ -1954,6 +2165,29 @@ unsigned int xa_extract(struct xarray *xa, void **dst, unsigned long start, EXPORT_SYMBOL(xa_extract); /** + * xa_delete_node() - Private interface for workingset code. + * @node: Node to be removed from the tree. + * @update: Function to call to update ancestor nodes. + * + * Context: xa_lock must be held on entry and will not be released. + */ +void xa_delete_node(struct xa_node *node, xa_update_node_t update) +{ + struct xa_state xas = { + .xa = node->array, + .xa_index = (unsigned long)node->offset << + (node->shift + XA_CHUNK_SHIFT), + .xa_shift = node->shift + XA_CHUNK_SHIFT, + .xa_offset = node->offset, + .xa_node = xa_parent_locked(node->array, node), + .xa_update = update, + }; + + xas_store(&xas, NULL); +} +EXPORT_SYMBOL_GPL(xa_delete_node); /* For the benefit of the test suite */ + +/** * xa_destroy() - Free all internal data structures. * @xa: XArray. * diff --git a/lib/xxhash.c b/lib/xxhash.c index aa61e2a3802f..d5bb9ff10607 100644 --- a/lib/xxhash.c +++ b/lib/xxhash.c @@ -34,7 +34,7 @@ * ("BSD"). * * You can contact the author at: - * - xxHash homepage: http://cyan4973.github.io/xxHash/ + * - xxHash homepage: https://cyan4973.github.io/xxHash/ * - xxHash source repository: https://github.com/Cyan4973/xxHash */ diff --git a/lib/xz/Kconfig b/lib/xz/Kconfig index 22528743d4ce..adce22ac18d6 100644 --- a/lib/xz/Kconfig +++ b/lib/xz/Kconfig @@ -5,7 +5,7 @@ config XZ_DEC help LZMA2 compression algorithm and BCJ filters are supported using the .xz file format as the container. For integrity checking, - CRC32 is supported. See Documentation/xz.txt for more information. + CRC32 is supported. See Documentation/staging/xz.rst for more information. if XZ_DEC @@ -39,6 +39,19 @@ config XZ_DEC_SPARC default y select XZ_DEC_BCJ +config XZ_DEC_MICROLZMA + bool "MicroLZMA decoder" + default n + help + MicroLZMA is a header format variant where the first byte + of a raw LZMA stream (without the end of stream marker) has + been replaced with a bitwise-negation of the lc/lp/pb + properties byte. MicroLZMA was created to be used in EROFS + but can be used by other things too where wasting minimal + amount of space for headers is important. + + Unless you know that you need this, say N. + endif config XZ_DEC_BCJ diff --git a/lib/xz/xz_crc32.c b/lib/xz/xz_crc32.c index 912aae5fa09e..88a2c35e1b59 100644 --- a/lib/xz/xz_crc32.c +++ b/lib/xz/xz_crc32.c @@ -2,7 +2,7 @@ * CRC32 using the polynomial from IEEE-802.3 * * Authors: Lasse Collin <lasse.collin@tukaani.org> - * Igor Pavlov <http://7-zip.org/> + * Igor Pavlov <https://7-zip.org/> * * This file has been put into the public domain. * You can do whatever you want with this file. diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index a768e6d28bbb..ef449e97d1a1 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -2,7 +2,7 @@ * Branch/Call/Jump (BCJ) filter decoders * * Authors: Lasse Collin <lasse.collin@tukaani.org> - * Igor Pavlov <http://7-zip.org/> + * Igor Pavlov <https://7-zip.org/> * * This file has been put into the public domain. * You can do whatever you want with this file. @@ -422,7 +422,7 @@ XZ_EXTERN enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s, /* * Flush pending already filtered data to the output buffer. Return - * immediatelly if we couldn't flush everything, or if the next + * immediately if we couldn't flush everything, or if the next * filter in the chain had already returned XZ_STREAM_END. */ if (s->temp.filtered > 0) { diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c index 156f26fdc4c9..27ce34520e78 100644 --- a/lib/xz/xz_dec_lzma2.c +++ b/lib/xz/xz_dec_lzma2.c @@ -2,7 +2,7 @@ * LZMA2 decoder * * Authors: Lasse Collin <lasse.collin@tukaani.org> - * Igor Pavlov <http://7-zip.org/> + * Igor Pavlov <https://7-zip.org/> * * This file has been put into the public domain. * You can do whatever you want with this file. @@ -147,8 +147,8 @@ struct lzma_dec { /* * LZMA properties or related bit masks (number of literal - * context bits, a mask dervied from the number of literal - * position bits, and a mask dervied from the number + * context bits, a mask derived from the number of literal + * position bits, and a mask derived from the number * position bits) */ uint32_t lc; @@ -248,6 +248,10 @@ struct lzma2_dec { * before the first LZMA chunk. */ bool need_props; + +#ifdef XZ_DEC_MICROLZMA + bool pedantic_microlzma; +#endif }; struct xz_dec_lzma2 { @@ -387,7 +391,14 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, *left -= copy_size; - memcpy(dict->buf + dict->pos, b->in + b->in_pos, copy_size); + /* + * If doing in-place decompression in single-call mode and the + * uncompressed size of the file is larger than the caller + * thought (i.e. it is invalid input!), the buffers below may + * overlap and cause undefined behavior with memcpy(). + * With valid inputs memcpy() would be fine here. + */ + memmove(dict->buf + dict->pos, b->in + b->in_pos, copy_size); dict->pos += copy_size; if (dict->full < dict->pos) @@ -397,7 +408,11 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, if (dict->pos == dict->end) dict->pos = 0; - memcpy(b->out + b->out_pos, b->in + b->in_pos, + /* + * Like above but for multi-call mode: use memmove() + * to avoid undefined behavior with invalid input. + */ + memmove(b->out + b->out_pos, b->in + b->in_pos, copy_size); } @@ -408,6 +423,12 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, } } +#ifdef XZ_DEC_MICROLZMA +# define DICT_FLUSH_SUPPORTS_SKIPPING true +#else +# define DICT_FLUSH_SUPPORTS_SKIPPING false +#endif + /* * Flush pending data from dictionary to b->out. It is assumed that there is * enough space in b->out. This is guaranteed because caller uses dict_limit() @@ -421,8 +442,19 @@ static uint32_t dict_flush(struct dictionary *dict, struct xz_buf *b) if (dict->pos == dict->end) dict->pos = 0; - memcpy(b->out + b->out_pos, dict->buf + dict->start, - copy_size); + /* + * These buffers cannot overlap even if doing in-place + * decompression because in multi-call mode dict->buf + * has been allocated by us in this file; it's not + * provided by the caller like in single-call mode. + * + * With MicroLZMA, b->out can be NULL to skip bytes that + * the caller doesn't need. This cannot be done with XZ + * because it would break BCJ filters. + */ + if (!DICT_FLUSH_SUPPORTS_SKIPPING || b->out != NULL) + memcpy(b->out + b->out_pos, dict->buf + dict->start, + copy_size); } dict->start = dict->pos; @@ -484,11 +516,11 @@ static __always_inline void rc_normalize(struct rc_dec *rc) } /* - * Decode one bit. In some versions, this function has been splitted in three + * Decode one bit. In some versions, this function has been split in three * functions so that the compiler is supposed to be able to more easily avoid * an extra branch. In this particular version of the LZMA decoder, this * doesn't seem to be a good idea (tested with GCC 3.3.6, 3.4.6, and 4.3.3 - * on x86). Using a non-splitted version results in nicer looking code too. + * on x86). Using a non-split version results in nicer looking code too. * * NOTE: This must return an int. Do not make it return a bool or the speed * of the code generated by GCC 3.x decreases 10-15 %. (GCC 4.3 doesn't care, @@ -761,7 +793,7 @@ static bool lzma_main(struct xz_dec_lzma2 *s) } /* - * Reset the LZMA decoder and range decoder state. Dictionary is nore reset + * Reset the LZMA decoder and range decoder state. Dictionary is not reset * here, because LZMA state may be reset without resetting the dictionary. */ static void lzma_reset(struct xz_dec_lzma2 *s) @@ -774,6 +806,7 @@ static void lzma_reset(struct xz_dec_lzma2 *s) s->lzma.rep1 = 0; s->lzma.rep2 = 0; s->lzma.rep3 = 0; + s->lzma.len = 0; /* * All probabilities are initialized to the same value. This hack @@ -1043,7 +1076,7 @@ XZ_EXTERN enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, s->lzma2.sequence = SEQ_LZMA_PREPARE; - /* Fall through */ + fallthrough; case SEQ_LZMA_PREPARE: if (s->lzma2.compressed < RC_INIT_BYTES) @@ -1055,7 +1088,7 @@ XZ_EXTERN enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, s->lzma2.compressed -= RC_INIT_BYTES; s->lzma2.sequence = SEQ_LZMA_RUN; - /* Fall through */ + fallthrough; case SEQ_LZMA_RUN: /* @@ -1157,8 +1190,6 @@ XZ_EXTERN enum xz_ret xz_dec_lzma2_reset(struct xz_dec_lzma2 *s, uint8_t props) } } - s->lzma.len = 0; - s->lzma2.sequence = SEQ_CONTROL; s->lzma2.need_dict_reset = true; @@ -1174,3 +1205,140 @@ XZ_EXTERN void xz_dec_lzma2_end(struct xz_dec_lzma2 *s) kfree(s); } + +#ifdef XZ_DEC_MICROLZMA +/* This is a wrapper struct to have a nice struct name in the public API. */ +struct xz_dec_microlzma { + struct xz_dec_lzma2 s; +}; + +enum xz_ret xz_dec_microlzma_run(struct xz_dec_microlzma *s_ptr, + struct xz_buf *b) +{ + struct xz_dec_lzma2 *s = &s_ptr->s; + + /* + * sequence is SEQ_PROPERTIES before the first input byte, + * SEQ_LZMA_PREPARE until a total of five bytes have been read, + * and SEQ_LZMA_RUN for the rest of the input stream. + */ + if (s->lzma2.sequence != SEQ_LZMA_RUN) { + if (s->lzma2.sequence == SEQ_PROPERTIES) { + /* One byte is needed for the props. */ + if (b->in_pos >= b->in_size) + return XZ_OK; + + /* + * Don't increment b->in_pos here. The same byte is + * also passed to rc_read_init() which will ignore it. + */ + if (!lzma_props(s, ~b->in[b->in_pos])) + return XZ_DATA_ERROR; + + s->lzma2.sequence = SEQ_LZMA_PREPARE; + } + + /* + * xz_dec_microlzma_reset() doesn't validate the compressed + * size so we do it here. We have to limit the maximum size + * to avoid integer overflows in lzma2_lzma(). 3 GiB is a nice + * round number and much more than users of this code should + * ever need. + */ + if (s->lzma2.compressed < RC_INIT_BYTES + || s->lzma2.compressed > (3U << 30)) + return XZ_DATA_ERROR; + + if (!rc_read_init(&s->rc, b)) + return XZ_OK; + + s->lzma2.compressed -= RC_INIT_BYTES; + s->lzma2.sequence = SEQ_LZMA_RUN; + + dict_reset(&s->dict, b); + } + + /* This is to allow increasing b->out_size between calls. */ + if (DEC_IS_SINGLE(s->dict.mode)) + s->dict.end = b->out_size - b->out_pos; + + while (true) { + dict_limit(&s->dict, min_t(size_t, b->out_size - b->out_pos, + s->lzma2.uncompressed)); + + if (!lzma2_lzma(s, b)) + return XZ_DATA_ERROR; + + s->lzma2.uncompressed -= dict_flush(&s->dict, b); + + if (s->lzma2.uncompressed == 0) { + if (s->lzma2.pedantic_microlzma) { + if (s->lzma2.compressed > 0 || s->lzma.len > 0 + || !rc_is_finished(&s->rc)) + return XZ_DATA_ERROR; + } + + return XZ_STREAM_END; + } + + if (b->out_pos == b->out_size) + return XZ_OK; + + if (b->in_pos == b->in_size + && s->temp.size < s->lzma2.compressed) + return XZ_OK; + } +} + +struct xz_dec_microlzma *xz_dec_microlzma_alloc(enum xz_mode mode, + uint32_t dict_size) +{ + struct xz_dec_microlzma *s; + + /* Restrict dict_size to the same range as in the LZMA2 code. */ + if (dict_size < 4096 || dict_size > (3U << 30)) + return NULL; + + s = kmalloc(sizeof(*s), GFP_KERNEL); + if (s == NULL) + return NULL; + + s->s.dict.mode = mode; + s->s.dict.size = dict_size; + + if (DEC_IS_MULTI(mode)) { + s->s.dict.end = dict_size; + + s->s.dict.buf = vmalloc(dict_size); + if (s->s.dict.buf == NULL) { + kfree(s); + return NULL; + } + } + + return s; +} + +void xz_dec_microlzma_reset(struct xz_dec_microlzma *s, uint32_t comp_size, + uint32_t uncomp_size, int uncomp_size_is_exact) +{ + /* + * comp_size is validated in xz_dec_microlzma_run(). + * uncomp_size can safely be anything. + */ + s->s.lzma2.compressed = comp_size; + s->s.lzma2.uncompressed = uncomp_size; + s->s.lzma2.pedantic_microlzma = uncomp_size_is_exact; + + s->s.lzma2.sequence = SEQ_PROPERTIES; + s->s.temp.size = 0; +} + +void xz_dec_microlzma_end(struct xz_dec_microlzma *s) +{ + if (DEC_IS_MULTI(s->s.dict.mode)) + vfree(s->s.dict.buf); + + kfree(s); +} +#endif diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c index bd1d182419d7..683570b93a8c 100644 --- a/lib/xz/xz_dec_stream.c +++ b/lib/xz/xz_dec_stream.c @@ -402,12 +402,12 @@ static enum xz_ret dec_stream_header(struct xz_dec *s) * we will accept other check types too, but then the check won't * be verified and a warning (XZ_UNSUPPORTED_CHECK) will be given. */ + if (s->temp.buf[HEADER_MAGIC_SIZE + 1] > XZ_CHECK_MAX) + return XZ_OPTIONS_ERROR; + s->check_type = s->temp.buf[HEADER_MAGIC_SIZE + 1]; #ifdef XZ_DEC_ANY_CHECK - if (s->check_type > XZ_CHECK_MAX) - return XZ_OPTIONS_ERROR; - if (s->check_type > XZ_CHECK_CRC32) return XZ_UNSUPPORTED_CHECK; #else @@ -583,7 +583,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) if (ret != XZ_OK) return ret; - /* Fall through */ + fallthrough; case SEQ_BLOCK_START: /* We need one byte of input to continue. */ @@ -608,7 +608,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->temp.pos = 0; s->sequence = SEQ_BLOCK_HEADER; - /* Fall through */ + fallthrough; case SEQ_BLOCK_HEADER: if (!fill_temp(s, b)) @@ -620,7 +620,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->sequence = SEQ_BLOCK_UNCOMPRESS; - /* Fall through */ + fallthrough; case SEQ_BLOCK_UNCOMPRESS: ret = dec_block(s, b); @@ -629,7 +629,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->sequence = SEQ_BLOCK_PADDING; - /* Fall through */ + fallthrough; case SEQ_BLOCK_PADDING: /* @@ -651,7 +651,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->sequence = SEQ_BLOCK_CHECK; - /* Fall through */ + fallthrough; case SEQ_BLOCK_CHECK: if (s->check_type == XZ_CHECK_CRC32) { @@ -675,7 +675,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->sequence = SEQ_INDEX_PADDING; - /* Fall through */ + fallthrough; case SEQ_INDEX_PADDING: while ((s->index.size + (b->in_pos - s->in_start)) @@ -699,7 +699,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->sequence = SEQ_INDEX_CRC32; - /* Fall through */ + fallthrough; case SEQ_INDEX_CRC32: ret = crc32_validate(s, b); @@ -709,7 +709,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->temp.size = STREAM_HEADER_SIZE; s->sequence = SEQ_STREAM_FOOTER; - /* Fall through */ + fallthrough; case SEQ_STREAM_FOOTER: if (!fill_temp(s, b)) diff --git a/lib/xz/xz_dec_syms.c b/lib/xz/xz_dec_syms.c index 32eb3c03aede..61098c67a413 100644 --- a/lib/xz/xz_dec_syms.c +++ b/lib/xz/xz_dec_syms.c @@ -15,8 +15,15 @@ EXPORT_SYMBOL(xz_dec_reset); EXPORT_SYMBOL(xz_dec_run); EXPORT_SYMBOL(xz_dec_end); +#ifdef CONFIG_XZ_DEC_MICROLZMA +EXPORT_SYMBOL(xz_dec_microlzma_alloc); +EXPORT_SYMBOL(xz_dec_microlzma_reset); +EXPORT_SYMBOL(xz_dec_microlzma_run); +EXPORT_SYMBOL(xz_dec_microlzma_end); +#endif + MODULE_DESCRIPTION("XZ decompressor"); -MODULE_VERSION("1.0"); +MODULE_VERSION("1.1"); MODULE_AUTHOR("Lasse Collin <lasse.collin@tukaani.org> and Igor Pavlov"); /* diff --git a/lib/xz/xz_lzma2.h b/lib/xz/xz_lzma2.h index 071d67bee9f5..92d852d4f87a 100644 --- a/lib/xz/xz_lzma2.h +++ b/lib/xz/xz_lzma2.h @@ -2,7 +2,7 @@ * LZMA2 definitions * * Authors: Lasse Collin <lasse.collin@tukaani.org> - * Igor Pavlov <http://7-zip.org/> + * Igor Pavlov <https://7-zip.org/> * * This file has been put into the public domain. * You can do whatever you want with this file. diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h index 09360ebb510e..bf1e94ec7873 100644 --- a/lib/xz/xz_private.h +++ b/lib/xz/xz_private.h @@ -37,6 +37,9 @@ # ifdef CONFIG_XZ_DEC_SPARC # define XZ_DEC_SPARC # endif +# ifdef CONFIG_XZ_DEC_MICROLZMA +# define XZ_DEC_MICROLZMA +# endif # define memeq(a, b, size) (memcmp(a, b, size) == 0) # define memzero(buf, size) memset(buf, 0, size) # endif diff --git a/lib/xz/xz_stream.h b/lib/xz/xz_stream.h index 66cb5a7055ec..430bb3a0d195 100644 --- a/lib/xz/xz_stream.h +++ b/lib/xz/xz_stream.h @@ -19,7 +19,7 @@ /* * See the .xz file format specification at - * http://tukaani.org/xz/xz-file-format.txt + * https://tukaani.org/xz/xz-file-format.txt * to understand the container format. */ diff --git a/lib/zlib_deflate/deflate.c b/lib/zlib_deflate/deflate.c index d20ef458f137..8a878d0d892c 100644 --- a/lib/zlib_deflate/deflate.c +++ b/lib/zlib_deflate/deflate.c @@ -52,16 +52,19 @@ #include <linux/zutil.h> #include "defutil.h" +/* architecture-specific bits */ +#ifdef CONFIG_ZLIB_DFLTCC +# include "../zlib_dfltcc/dfltcc.h" +#else +#define DEFLATE_RESET_HOOK(strm) do {} while (0) +#define DEFLATE_HOOK(strm, flush, bstate) 0 +#define DEFLATE_NEED_CHECKSUM(strm) 1 +#define DEFLATE_DFLTCC_ENABLED() 0 +#endif /* =========================================================================== * Function prototypes. */ -typedef enum { - need_more, /* block not completed, need more input or more output */ - block_done, /* block flush performed */ - finish_started, /* finish started, need only more output at next deflate */ - finish_done /* finish done, accept no more input or output */ -} block_state; typedef block_state (*compress_func) (deflate_state *s, int flush); /* Compression function. Returns the block state after the call. */ @@ -72,7 +75,6 @@ static block_state deflate_fast (deflate_state *s, int flush); static block_state deflate_slow (deflate_state *s, int flush); static void lm_init (deflate_state *s); static void putShortMSB (deflate_state *s, uInt b); -static void flush_pending (z_streamp strm); static int read_buf (z_streamp strm, Byte *buf, unsigned size); static uInt longest_match (deflate_state *s, IPos cur_match); @@ -98,6 +100,25 @@ static void check_match (deflate_state *s, IPos start, IPos match, * See deflate.c for comments about the MIN_MATCH+1. */ +/* Workspace to be allocated for deflate processing */ +typedef struct deflate_workspace { + /* State memory for the deflator */ + deflate_state deflate_memory; +#ifdef CONFIG_ZLIB_DFLTCC + /* State memory for s390 hardware deflate */ + struct dfltcc_state dfltcc_memory; +#endif + Byte *window_memory; + Pos *prev_memory; + Pos *head_memory; + char *overlay_memory; +} deflate_workspace; + +#ifdef CONFIG_ZLIB_DFLTCC +/* dfltcc_state must be doubleword aligned for DFLTCC call */ +static_assert(offsetof(struct deflate_workspace, dfltcc_memory) % 8 == 0); +#endif + /* Values for max_lazy_match, good_match and max_chain_length, depending on * the desired pack level (0..9). The values given below have been tuned to * exclude worst case performance for pathological files. Better values may be @@ -207,7 +228,15 @@ int zlib_deflateInit2( */ next = (char *) mem; next += sizeof(*mem); +#ifdef CONFIG_ZLIB_DFLTCC + /* + * DFLTCC requires the window to be page aligned. + * Thus, we overallocate and take the aligned portion of the buffer. + */ + mem->window_memory = (Byte *) PTR_ALIGN(next, PAGE_SIZE); +#else mem->window_memory = (Byte *) next; +#endif next += zlib_deflate_window_memsize(windowBits); mem->prev_memory = (Pos *) next; next += zlib_deflate_prev_memsize(windowBits); @@ -277,6 +306,8 @@ int zlib_deflateReset( zlib_tr_init(s); lm_init(s); + DEFLATE_RESET_HOOK(strm); + return Z_OK; } @@ -294,35 +325,6 @@ static void putShortMSB( put_byte(s, (Byte)(b & 0xff)); } -/* ========================================================================= - * Flush as much pending output as possible. All deflate() output goes - * through this function so some applications may wish to modify it - * to avoid allocating a large strm->next_out buffer and copying into it. - * (See also read_buf()). - */ -static void flush_pending( - z_streamp strm -) -{ - deflate_state *s = (deflate_state *) strm->state; - unsigned len = s->pending; - - if (len > strm->avail_out) len = strm->avail_out; - if (len == 0) return; - - if (strm->next_out != NULL) { - memcpy(strm->next_out, s->pending_out, len); - strm->next_out += len; - } - s->pending_out += len; - strm->total_out += len; - strm->avail_out -= len; - s->pending -= len; - if (s->pending == 0) { - s->pending_out = s->pending_buf; - } -} - /* ========================================================================= */ int zlib_deflate( z_streamp strm, @@ -404,7 +406,8 @@ int zlib_deflate( (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { block_state bstate; - bstate = (*(configuration_table[s->level].func))(s, flush); + bstate = DEFLATE_HOOK(strm, flush, &bstate) ? bstate : + (*(configuration_table[s->level].func))(s, flush); if (bstate == finish_started || bstate == finish_done) { s->status = FINISH_STATE; @@ -503,7 +506,8 @@ static int read_buf( strm->avail_in -= len; - if (!((deflate_state *)(strm->state))->noheader) { + if (!DEFLATE_NEED_CHECKSUM(strm)) {} + else if (!((deflate_state *)(strm->state))->noheader) { strm->adler = zlib_adler32(strm->adler, strm->next_in, len); } memcpy(buf, strm->next_in, len); @@ -1135,3 +1139,8 @@ int zlib_deflate_workspacesize(int windowBits, int memLevel) + zlib_deflate_head_memsize(memLevel) + zlib_deflate_overlay_memsize(memLevel); } + +int zlib_deflate_dfltcc_enabled(void) +{ + return DEFLATE_DFLTCC_ENABLED(); +} diff --git a/lib/zlib_deflate/deflate_syms.c b/lib/zlib_deflate/deflate_syms.c index 72fe4b73be53..24b740b99678 100644 --- a/lib/zlib_deflate/deflate_syms.c +++ b/lib/zlib_deflate/deflate_syms.c @@ -12,6 +12,7 @@ #include <linux/zlib.h> EXPORT_SYMBOL(zlib_deflate_workspacesize); +EXPORT_SYMBOL(zlib_deflate_dfltcc_enabled); EXPORT_SYMBOL(zlib_deflate); EXPORT_SYMBOL(zlib_deflateInit2); EXPORT_SYMBOL(zlib_deflateEnd); diff --git a/lib/zlib_deflate/deftree.c b/lib/zlib_deflate/deftree.c index 9b1756b12743..a4a34da512fe 100644 --- a/lib/zlib_deflate/deftree.c +++ b/lib/zlib_deflate/deftree.c @@ -76,11 +76,6 @@ static const uch bl_order[BL_CODES] * probability, to avoid transmitting the lengths for unused bit length codes. */ -#define Buf_size (8 * 2*sizeof(char)) -/* Number of bits used within bi_buf. (bi_buf might be implemented on - * more than 16 bits on some systems.) - */ - /* =========================================================================== * Local data. These are initialized only once. */ @@ -147,7 +142,6 @@ static void send_all_trees (deflate_state *s, int lcodes, int dcodes, static void compress_block (deflate_state *s, ct_data *ltree, ct_data *dtree); static void set_data_type (deflate_state *s); -static void bi_windup (deflate_state *s); static void bi_flush (deflate_state *s); static void copy_block (deflate_state *s, char *buf, unsigned len, int header); @@ -170,54 +164,6 @@ static void copy_block (deflate_state *s, char *buf, unsigned len, */ /* =========================================================================== - * Send a value on a given number of bits. - * IN assertion: length <= 16 and value fits in length bits. - */ -#ifdef DEBUG_ZLIB -static void send_bits (deflate_state *s, int value, int length); - -static void send_bits( - deflate_state *s, - int value, /* value to send */ - int length /* number of bits */ -) -{ - Tracevv((stderr," l %2d v %4x ", length, value)); - Assert(length > 0 && length <= 15, "invalid length"); - s->bits_sent += (ulg)length; - - /* If not enough room in bi_buf, use (valid) bits from bi_buf and - * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid)) - * unused bits in value. - */ - if (s->bi_valid > (int)Buf_size - length) { - s->bi_buf |= (value << s->bi_valid); - put_short(s, s->bi_buf); - s->bi_buf = (ush)value >> (Buf_size - s->bi_valid); - s->bi_valid += length - Buf_size; - } else { - s->bi_buf |= value << s->bi_valid; - s->bi_valid += length; - } -} -#else /* !DEBUG_ZLIB */ - -#define send_bits(s, value, length) \ -{ int len = length;\ - if (s->bi_valid > (int)Buf_size - len) {\ - int val = value;\ - s->bi_buf |= (val << s->bi_valid);\ - put_short(s, s->bi_buf);\ - s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ - s->bi_valid += len - Buf_size;\ - } else {\ - s->bi_buf |= (value) << s->bi_valid;\ - s->bi_valid += len;\ - }\ -} -#endif /* DEBUG_ZLIB */ - -/* =========================================================================== * Initialize the various 'constant' tables. In a multi-threaded environment, * this function may be called by two threads concurrently, but this is * harmless since both invocations do exactly the same thing. diff --git a/lib/zlib_deflate/defutil.h b/lib/zlib_deflate/defutil.h index a8c370897c9f..385333b22ec6 100644 --- a/lib/zlib_deflate/defutil.h +++ b/lib/zlib_deflate/defutil.h @@ -1,5 +1,7 @@ +#ifndef DEFUTIL_H +#define DEFUTIL_H - +#include <linux/zutil.h> #define Assert(err, str) #define Trace(dummy) @@ -238,17 +240,13 @@ typedef struct deflate_state { } deflate_state; -typedef struct deflate_workspace { - /* State memory for the deflator */ - deflate_state deflate_memory; - Byte *window_memory; - Pos *prev_memory; - Pos *head_memory; - char *overlay_memory; -} deflate_workspace; - +#ifdef CONFIG_ZLIB_DFLTCC +#define zlib_deflate_window_memsize(windowBits) \ + (2 * (1 << (windowBits)) * sizeof(Byte) + PAGE_SIZE) +#else #define zlib_deflate_window_memsize(windowBits) \ (2 * (1 << (windowBits)) * sizeof(Byte)) +#endif #define zlib_deflate_prev_memsize(windowBits) \ ((1 << (windowBits)) * sizeof(Pos)) #define zlib_deflate_head_memsize(memLevel) \ @@ -293,6 +291,24 @@ void zlib_tr_stored_type_only (deflate_state *); } /* =========================================================================== + * Reverse the first len bits of a code, using straightforward code (a faster + * method would use a table) + * IN assertion: 1 <= len <= 15 + */ +static inline unsigned bi_reverse( + unsigned code, /* the value to invert */ + int len /* its bit length */ +) +{ + register unsigned res = 0; + do { + res |= code & 1; + code >>= 1, res <<= 1; + } while (--len > 0); + return res >> 1; +} + +/* =========================================================================== * Flush the bit buffer, keeping at most 7 bits in it. */ static inline void bi_flush(deflate_state *s) @@ -325,3 +341,101 @@ static inline void bi_windup(deflate_state *s) #endif } +typedef enum { + need_more, /* block not completed, need more input or more output */ + block_done, /* block flush performed */ + finish_started, /* finish started, need only more output at next deflate */ + finish_done /* finish done, accept no more input or output */ +} block_state; + +#define Buf_size (8 * 2*sizeof(char)) +/* Number of bits used within bi_buf. (bi_buf might be implemented on + * more than 16 bits on some systems.) + */ + +/* =========================================================================== + * Send a value on a given number of bits. + * IN assertion: length <= 16 and value fits in length bits. + */ +#ifdef DEBUG_ZLIB +static void send_bits (deflate_state *s, int value, int length); + +static void send_bits( + deflate_state *s, + int value, /* value to send */ + int length /* number of bits */ +) +{ + Tracevv((stderr," l %2d v %4x ", length, value)); + Assert(length > 0 && length <= 15, "invalid length"); + s->bits_sent += (ulg)length; + + /* If not enough room in bi_buf, use (valid) bits from bi_buf and + * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid)) + * unused bits in value. + */ + if (s->bi_valid > (int)Buf_size - length) { + s->bi_buf |= (value << s->bi_valid); + put_short(s, s->bi_buf); + s->bi_buf = (ush)value >> (Buf_size - s->bi_valid); + s->bi_valid += length - Buf_size; + } else { + s->bi_buf |= value << s->bi_valid; + s->bi_valid += length; + } +} +#else /* !DEBUG_ZLIB */ + +#define send_bits(s, value, length) \ +{ int len = length;\ + if (s->bi_valid > (int)Buf_size - len) {\ + int val = value;\ + s->bi_buf |= (val << s->bi_valid);\ + put_short(s, s->bi_buf);\ + s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ + s->bi_valid += len - Buf_size;\ + } else {\ + s->bi_buf |= (value) << s->bi_valid;\ + s->bi_valid += len;\ + }\ +} +#endif /* DEBUG_ZLIB */ + +static inline void zlib_tr_send_bits( + deflate_state *s, + int value, + int length +) +{ + send_bits(s, value, length); +} + +/* ========================================================================= + * Flush as much pending output as possible. All deflate() output goes + * through this function so some applications may wish to modify it + * to avoid allocating a large strm->next_out buffer and copying into it. + * (See also read_buf()). + */ +static inline void flush_pending( + z_streamp strm +) +{ + deflate_state *s = (deflate_state *) strm->state; + unsigned len = s->pending; + + if (len > strm->avail_out) len = strm->avail_out; + if (len == 0) return; + + if (strm->next_out != NULL) { + memcpy(strm->next_out, s->pending_out, len); + strm->next_out += len; + } + s->pending_out += len; + strm->total_out += len; + strm->avail_out -= len; + s->pending -= len; + if (s->pending == 0) { + s->pending_out = s->pending_buf; + } +} +#endif /* DEFUTIL_H */ diff --git a/lib/zlib_dfltcc/Makefile b/lib/zlib_dfltcc/Makefile new file mode 100644 index 000000000000..66e1c96387c4 --- /dev/null +++ b/lib/zlib_dfltcc/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This is a modified version of zlib, which does all memory +# allocation ahead of time. +# +# This is the code for s390 zlib hardware support. +# + +obj-$(CONFIG_ZLIB_DFLTCC) += zlib_dfltcc.o + +zlib_dfltcc-objs := dfltcc.o dfltcc_deflate.o dfltcc_inflate.o diff --git a/lib/zlib_dfltcc/dfltcc.c b/lib/zlib_dfltcc/dfltcc.c new file mode 100644 index 000000000000..782f76e9d4da --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Zlib +/* dfltcc.c - SystemZ DEFLATE CONVERSION CALL support. */ + +#include <linux/export.h> +#include <linux/module.h> +#include "dfltcc_util.h" +#include "dfltcc.h" + +char *oesc_msg( + char *buf, + int oesc +) +{ + if (oesc == 0x00) + return NULL; /* Successful completion */ + else { +#ifdef STATIC + return NULL; /* Ignore for pre-boot decompressor */ +#else + sprintf(buf, "Operation-Ending-Supplemental Code is 0x%.2X", oesc); + return buf; +#endif + } +} + +void dfltcc_reset( + z_streamp strm, + uInt size +) +{ + struct dfltcc_state *dfltcc_state = + (struct dfltcc_state *)((char *)strm->state + size); + struct dfltcc_qaf_param *param = + (struct dfltcc_qaf_param *)&dfltcc_state->param; + + /* Initialize available functions */ + if (is_dfltcc_enabled()) { + dfltcc(DFLTCC_QAF, param, NULL, NULL, NULL, NULL, NULL); + memmove(&dfltcc_state->af, param, sizeof(dfltcc_state->af)); + } else + memset(&dfltcc_state->af, 0, sizeof(dfltcc_state->af)); + + /* Initialize parameter block */ + memset(&dfltcc_state->param, 0, sizeof(dfltcc_state->param)); + dfltcc_state->param.nt = 1; + + /* Initialize tuning parameters */ + if (zlib_dfltcc_support == ZLIB_DFLTCC_FULL_DEBUG) + dfltcc_state->level_mask = DFLTCC_LEVEL_MASK_DEBUG; + else + dfltcc_state->level_mask = DFLTCC_LEVEL_MASK; + dfltcc_state->block_size = DFLTCC_BLOCK_SIZE; + dfltcc_state->block_threshold = DFLTCC_FIRST_FHT_BLOCK_SIZE; + dfltcc_state->dht_threshold = DFLTCC_DHT_MIN_SAMPLE_SIZE; + dfltcc_state->param.ribm = DFLTCC_RIBM; +} +EXPORT_SYMBOL(dfltcc_reset); + +MODULE_LICENSE("GPL"); diff --git a/lib/zlib_dfltcc/dfltcc.h b/lib/zlib_dfltcc/dfltcc.h new file mode 100644 index 000000000000..2a2fac1d050a --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc.h @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: Zlib +#ifndef DFLTCC_H +#define DFLTCC_H + +#include "../zlib_deflate/defutil.h" +#include <asm/facility.h> +#include <asm/setup.h> + +/* + * Tuning parameters. + */ +#define DFLTCC_LEVEL_MASK 0x2 /* DFLTCC compression for level 1 only */ +#define DFLTCC_LEVEL_MASK_DEBUG 0x3fe /* DFLTCC compression for all levels */ +#define DFLTCC_BLOCK_SIZE 1048576 +#define DFLTCC_FIRST_FHT_BLOCK_SIZE 4096 +#define DFLTCC_DHT_MIN_SAMPLE_SIZE 4096 +#define DFLTCC_RIBM 0 + +#define DFLTCC_FACILITY 151 + +/* + * Parameter Block for Query Available Functions. + */ +struct dfltcc_qaf_param { + char fns[16]; + char reserved1[8]; + char fmts[2]; + char reserved2[6]; +}; + +static_assert(sizeof(struct dfltcc_qaf_param) == 32); + +#define DFLTCC_FMT0 0 + +/* + * Parameter Block for Generate Dynamic-Huffman Table, Compress and Expand. + */ +struct dfltcc_param_v0 { + uint16_t pbvn; /* Parameter-Block-Version Number */ + uint8_t mvn; /* Model-Version Number */ + uint8_t ribm; /* Reserved for IBM use */ + unsigned reserved32 : 31; + unsigned cf : 1; /* Continuation Flag */ + uint8_t reserved64[8]; + unsigned nt : 1; /* New Task */ + unsigned reserved129 : 1; + unsigned cvt : 1; /* Check Value Type */ + unsigned reserved131 : 1; + unsigned htt : 1; /* Huffman-Table Type */ + unsigned bcf : 1; /* Block-Continuation Flag */ + unsigned bcc : 1; /* Block Closing Control */ + unsigned bhf : 1; /* Block Header Final */ + unsigned reserved136 : 1; + unsigned reserved137 : 1; + unsigned dhtgc : 1; /* DHT Generation Control */ + unsigned reserved139 : 5; + unsigned reserved144 : 5; + unsigned sbb : 3; /* Sub-Byte Boundary */ + uint8_t oesc; /* Operation-Ending-Supplemental Code */ + unsigned reserved160 : 12; + unsigned ifs : 4; /* Incomplete-Function Status */ + uint16_t ifl; /* Incomplete-Function Length */ + uint8_t reserved192[8]; + uint8_t reserved256[8]; + uint8_t reserved320[4]; + uint16_t hl; /* History Length */ + unsigned reserved368 : 1; + uint16_t ho : 15; /* History Offset */ + uint32_t cv; /* Check Value */ + unsigned eobs : 15; /* End-of-block Symbol */ + unsigned reserved431: 1; + uint8_t eobl : 4; /* End-of-block Length */ + unsigned reserved436 : 12; + unsigned reserved448 : 4; + uint16_t cdhtl : 12; /* Compressed-Dynamic-Huffman Table + Length */ + uint8_t reserved464[6]; + uint8_t cdht[288]; + uint8_t reserved[32]; + uint8_t csb[1152]; +}; + +static_assert(sizeof(struct dfltcc_param_v0) == 1536); + +#define CVT_CRC32 0 +#define CVT_ADLER32 1 +#define HTT_FIXED 0 +#define HTT_DYNAMIC 1 + +/* + * Extension of inflate_state and deflate_state for DFLTCC. + */ +struct dfltcc_state { + struct dfltcc_param_v0 param; /* Parameter block */ + struct dfltcc_qaf_param af; /* Available functions */ + uLong level_mask; /* Levels on which to use DFLTCC */ + uLong block_size; /* New block each X bytes */ + uLong block_threshold; /* New block after total_in > X */ + uLong dht_threshold; /* New block only if avail_in >= X */ + char msg[64]; /* Buffer for strm->msg */ +}; + +/* Resides right after inflate_state or deflate_state */ +#define GET_DFLTCC_STATE(state) ((struct dfltcc_state *)((state) + 1)) + +/* External functions */ +int dfltcc_can_deflate(z_streamp strm); +int dfltcc_deflate(z_streamp strm, + int flush, + block_state *result); +void dfltcc_reset(z_streamp strm, uInt size); +int dfltcc_can_inflate(z_streamp strm); +typedef enum { + DFLTCC_INFLATE_CONTINUE, + DFLTCC_INFLATE_BREAK, + DFLTCC_INFLATE_SOFTWARE, +} dfltcc_inflate_action; +dfltcc_inflate_action dfltcc_inflate(z_streamp strm, + int flush, int *ret); +static inline int is_dfltcc_enabled(void) +{ +return (zlib_dfltcc_support != ZLIB_DFLTCC_DISABLED && + test_facility(DFLTCC_FACILITY)); +} + +#define DEFLATE_RESET_HOOK(strm) \ + dfltcc_reset((strm), sizeof(deflate_state)) + +#define DEFLATE_HOOK dfltcc_deflate + +#define DEFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_deflate((strm))) + +#define DEFLATE_DFLTCC_ENABLED() is_dfltcc_enabled() + +#define INFLATE_RESET_HOOK(strm) \ + dfltcc_reset((strm), sizeof(struct inflate_state)) + +#define INFLATE_TYPEDO_HOOK(strm, flush) \ + if (dfltcc_can_inflate((strm))) { \ + dfltcc_inflate_action action; \ +\ + RESTORE(); \ + action = dfltcc_inflate((strm), (flush), &ret); \ + LOAD(); \ + if (action == DFLTCC_INFLATE_CONTINUE) \ + break; \ + else if (action == DFLTCC_INFLATE_BREAK) \ + goto inf_leave; \ + } + +#define INFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_inflate((strm))) + +#define INFLATE_NEED_UPDATEWINDOW(strm) (!dfltcc_can_inflate((strm))) + +#endif /* DFLTCC_H */ diff --git a/lib/zlib_dfltcc/dfltcc_deflate.c b/lib/zlib_dfltcc/dfltcc_deflate.c new file mode 100644 index 000000000000..6c946e8532ee --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc_deflate.c @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: Zlib + +#include "../zlib_deflate/defutil.h" +#include "dfltcc_util.h" +#include "dfltcc.h" +#include <asm/setup.h> +#include <linux/export.h> +#include <linux/zutil.h> + +/* + * Compress. + */ +int dfltcc_can_deflate( + z_streamp strm +) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + + /* Check for kernel dfltcc command line parameter */ + if (zlib_dfltcc_support == ZLIB_DFLTCC_DISABLED || + zlib_dfltcc_support == ZLIB_DFLTCC_INFLATE_ONLY) + return 0; + + /* Unsupported compression settings */ + if (!dfltcc_are_params_ok(state->level, state->w_bits, state->strategy, + dfltcc_state->level_mask)) + return 0; + + /* Unsupported hardware */ + if (!is_bit_set(dfltcc_state->af.fns, DFLTCC_GDHT) || + !is_bit_set(dfltcc_state->af.fns, DFLTCC_CMPR) || + !is_bit_set(dfltcc_state->af.fmts, DFLTCC_FMT0)) + return 0; + + return 1; +} +EXPORT_SYMBOL(dfltcc_can_deflate); + +static void dfltcc_gdht( + z_streamp strm +) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + size_t avail_in = avail_in = strm->avail_in; + + dfltcc(DFLTCC_GDHT, + param, NULL, NULL, + &strm->next_in, &avail_in, NULL); +} + +static dfltcc_cc dfltcc_cmpr( + z_streamp strm +) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + size_t avail_in = strm->avail_in; + size_t avail_out = strm->avail_out; + dfltcc_cc cc; + + cc = dfltcc(DFLTCC_CMPR | HBT_CIRCULAR, + param, &strm->next_out, &avail_out, + &strm->next_in, &avail_in, state->window); + strm->total_in += (strm->avail_in - avail_in); + strm->total_out += (strm->avail_out - avail_out); + strm->avail_in = avail_in; + strm->avail_out = avail_out; + return cc; +} + +static void send_eobs( + z_streamp strm, + const struct dfltcc_param_v0 *param +) +{ + deflate_state *state = (deflate_state *)strm->state; + + zlib_tr_send_bits( + state, + bi_reverse(param->eobs >> (15 - param->eobl), param->eobl), + param->eobl); + flush_pending(strm); + if (state->pending != 0) { + /* The remaining data is located in pending_out[0:pending]. If someone + * calls put_byte() - this might happen in deflate() - the byte will be + * placed into pending_buf[pending], which is incorrect. Move the + * remaining data to the beginning of pending_buf so that put_byte() is + * usable again. + */ + memmove(state->pending_buf, state->pending_out, state->pending); + state->pending_out = state->pending_buf; + } +#ifdef ZLIB_DEBUG + state->compressed_len += param->eobl; +#endif +} + +int dfltcc_deflate( + z_streamp strm, + int flush, + block_state *result +) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + uInt masked_avail_in; + dfltcc_cc cc; + int need_empty_block; + int soft_bcc; + int no_flush; + + if (!dfltcc_can_deflate(strm)) + return 0; + +again: + masked_avail_in = 0; + soft_bcc = 0; + no_flush = flush == Z_NO_FLUSH; + + /* Trailing empty block. Switch to software, except when Continuation Flag + * is set, which means that DFLTCC has buffered some output in the + * parameter block and needs to be called again in order to flush it. + */ + if (flush == Z_FINISH && strm->avail_in == 0 && !param->cf) { + if (param->bcf) { + /* A block is still open, and the hardware does not support closing + * blocks without adding data. Thus, close it manually. + */ + send_eobs(strm, param); + param->bcf = 0; + } + return 0; + } + + if (strm->avail_in == 0 && !param->cf) { + *result = need_more; + return 1; + } + + /* There is an open non-BFINAL block, we are not going to close it just + * yet, we have compressed more than DFLTCC_BLOCK_SIZE bytes and we see + * more than DFLTCC_DHT_MIN_SAMPLE_SIZE bytes. Open a new block with a new + * DHT in order to adapt to a possibly changed input data distribution. + */ + if (param->bcf && no_flush && + strm->total_in > dfltcc_state->block_threshold && + strm->avail_in >= dfltcc_state->dht_threshold) { + if (param->cf) { + /* We need to flush the DFLTCC buffer before writing the + * End-of-block Symbol. Mask the input data and proceed as usual. + */ + masked_avail_in += strm->avail_in; + strm->avail_in = 0; + no_flush = 0; + } else { + /* DFLTCC buffer is empty, so we can manually write the + * End-of-block Symbol right away. + */ + send_eobs(strm, param); + param->bcf = 0; + dfltcc_state->block_threshold = + strm->total_in + dfltcc_state->block_size; + if (strm->avail_out == 0) { + *result = need_more; + return 1; + } + } + } + + /* The caller gave us too much data. Pass only one block worth of + * uncompressed data to DFLTCC and mask the rest, so that on the next + * iteration we start a new block. + */ + if (no_flush && strm->avail_in > dfltcc_state->block_size) { + masked_avail_in += (strm->avail_in - dfltcc_state->block_size); + strm->avail_in = dfltcc_state->block_size; + } + + /* When we have an open non-BFINAL deflate block and caller indicates that + * the stream is ending, we need to close an open deflate block and open a + * BFINAL one. + */ + need_empty_block = flush == Z_FINISH && param->bcf && !param->bhf; + + /* Translate stream to parameter block */ + param->cvt = CVT_ADLER32; + if (!no_flush) + /* We need to close a block. Always do this in software - when there is + * no input data, the hardware will not nohor BCC. */ + soft_bcc = 1; + if (flush == Z_FINISH && !param->bcf) + /* We are about to open a BFINAL block, set Block Header Final bit + * until the stream ends. + */ + param->bhf = 1; + /* DFLTCC-CMPR will write to next_out, so make sure that buffers with + * higher precedence are empty. + */ + Assert(state->pending == 0, "There must be no pending bytes"); + Assert(state->bi_valid < 8, "There must be less than 8 pending bits"); + param->sbb = (unsigned int)state->bi_valid; + if (param->sbb > 0) + *strm->next_out = (Byte)state->bi_buf; + if (param->hl) + param->nt = 0; /* Honor history */ + param->cv = strm->adler; + + /* When opening a block, choose a Huffman-Table Type */ + if (!param->bcf) { + if (strm->total_in == 0 && dfltcc_state->block_threshold > 0) { + param->htt = HTT_FIXED; + } + else { + param->htt = HTT_DYNAMIC; + dfltcc_gdht(strm); + } + } + + /* Deflate */ + do { + cc = dfltcc_cmpr(strm); + if (strm->avail_in < 4096 && masked_avail_in > 0) + /* We are about to call DFLTCC with a small input buffer, which is + * inefficient. Since there is masked data, there will be at least + * one more DFLTCC call, so skip the current one and make the next + * one handle more data. + */ + break; + } while (cc == DFLTCC_CC_AGAIN); + + /* Translate parameter block to stream */ + strm->msg = oesc_msg(dfltcc_state->msg, param->oesc); + state->bi_valid = param->sbb; + if (state->bi_valid == 0) + state->bi_buf = 0; /* Avoid accessing next_out */ + else + state->bi_buf = *strm->next_out & ((1 << state->bi_valid) - 1); + strm->adler = param->cv; + + /* Unmask the input data */ + strm->avail_in += masked_avail_in; + masked_avail_in = 0; + + /* If we encounter an error, it means there is a bug in DFLTCC call */ + Assert(cc != DFLTCC_CC_OP2_CORRUPT || param->oesc == 0, "BUG"); + + /* Update Block-Continuation Flag. It will be used to check whether to call + * GDHT the next time. + */ + if (cc == DFLTCC_CC_OK) { + if (soft_bcc) { + send_eobs(strm, param); + param->bcf = 0; + dfltcc_state->block_threshold = + strm->total_in + dfltcc_state->block_size; + } else + param->bcf = 1; + if (flush == Z_FINISH) { + if (need_empty_block) + /* Make the current deflate() call also close the stream */ + return 0; + else { + bi_windup(state); + *result = finish_done; + } + } else { + if (flush == Z_FULL_FLUSH) + param->hl = 0; /* Clear history */ + *result = flush == Z_NO_FLUSH ? need_more : block_done; + } + } else { + param->bcf = 1; + *result = need_more; + } + if (strm->avail_in != 0 && strm->avail_out != 0) + goto again; /* deflate() must use all input or all output */ + return 1; +} +EXPORT_SYMBOL(dfltcc_deflate); diff --git a/lib/zlib_dfltcc/dfltcc_inflate.c b/lib/zlib_dfltcc/dfltcc_inflate.c new file mode 100644 index 000000000000..fb60b5a6a1cb --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc_inflate.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Zlib + +#include "../zlib_inflate/inflate.h" +#include "dfltcc_util.h" +#include "dfltcc.h" +#include <asm/setup.h> +#include <linux/export.h> +#include <linux/zutil.h> + +/* + * Expand. + */ +int dfltcc_can_inflate( + z_streamp strm +) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + + /* Check for kernel dfltcc command line parameter */ + if (zlib_dfltcc_support == ZLIB_DFLTCC_DISABLED || + zlib_dfltcc_support == ZLIB_DFLTCC_DEFLATE_ONLY) + return 0; + + /* Unsupported compression settings */ + if (state->wbits != HB_BITS) + return 0; + + /* Unsupported hardware */ + return is_bit_set(dfltcc_state->af.fns, DFLTCC_XPND) && + is_bit_set(dfltcc_state->af.fmts, DFLTCC_FMT0); +} +EXPORT_SYMBOL(dfltcc_can_inflate); + +static int dfltcc_was_inflate_used( + z_streamp strm +) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + + return !param->nt; +} + +static int dfltcc_inflate_disable( + z_streamp strm +) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + + if (!dfltcc_can_inflate(strm)) + return 0; + if (dfltcc_was_inflate_used(strm)) + /* DFLTCC has already decompressed some data. Since there is not + * enough information to resume decompression in software, the call + * must fail. + */ + return 1; + /* DFLTCC was not used yet - decompress in software */ + memset(&dfltcc_state->af, 0, sizeof(dfltcc_state->af)); + return 0; +} + +static dfltcc_cc dfltcc_xpnd( + z_streamp strm +) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + size_t avail_in = strm->avail_in; + size_t avail_out = strm->avail_out; + dfltcc_cc cc; + + cc = dfltcc(DFLTCC_XPND | HBT_CIRCULAR, + param, &strm->next_out, &avail_out, + &strm->next_in, &avail_in, state->window); + strm->avail_in = avail_in; + strm->avail_out = avail_out; + return cc; +} + +dfltcc_inflate_action dfltcc_inflate( + z_streamp strm, + int flush, + int *ret +) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + dfltcc_cc cc; + + if (flush == Z_BLOCK) { + /* DFLTCC does not support stopping on block boundaries */ + if (dfltcc_inflate_disable(strm)) { + *ret = Z_STREAM_ERROR; + return DFLTCC_INFLATE_BREAK; + } else + return DFLTCC_INFLATE_SOFTWARE; + } + + if (state->last) { + if (state->bits != 0) { + strm->next_in++; + strm->avail_in--; + state->bits = 0; + } + state->mode = CHECK; + return DFLTCC_INFLATE_CONTINUE; + } + + if (strm->avail_in == 0 && !param->cf) + return DFLTCC_INFLATE_BREAK; + + if (!state->window || state->wsize == 0) { + state->mode = MEM; + return DFLTCC_INFLATE_CONTINUE; + } + + /* Translate stream to parameter block */ + param->cvt = CVT_ADLER32; + param->sbb = state->bits; + param->hl = state->whave; /* Software and hardware history formats match */ + param->ho = (state->write - state->whave) & ((1 << HB_BITS) - 1); + if (param->hl) + param->nt = 0; /* Honor history for the first block */ + param->cv = state->check; + + /* Inflate */ + do { + cc = dfltcc_xpnd(strm); + } while (cc == DFLTCC_CC_AGAIN); + + /* Translate parameter block to stream */ + strm->msg = oesc_msg(dfltcc_state->msg, param->oesc); + state->last = cc == DFLTCC_CC_OK; + state->bits = param->sbb; + state->whave = param->hl; + state->write = (param->ho + param->hl) & ((1 << HB_BITS) - 1); + state->check = param->cv; + if (cc == DFLTCC_CC_OP2_CORRUPT && param->oesc != 0) { + /* Report an error if stream is corrupted */ + state->mode = BAD; + return DFLTCC_INFLATE_CONTINUE; + } + state->mode = TYPEDO; + /* Break if operands are exhausted, otherwise continue looping */ + return (cc == DFLTCC_CC_OP1_TOO_SHORT || cc == DFLTCC_CC_OP2_TOO_SHORT) ? + DFLTCC_INFLATE_BREAK : DFLTCC_INFLATE_CONTINUE; +} +EXPORT_SYMBOL(dfltcc_inflate); diff --git a/lib/zlib_dfltcc/dfltcc_util.h b/lib/zlib_dfltcc/dfltcc_util.h new file mode 100644 index 000000000000..4a46b5009f0d --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc_util.h @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Zlib +#ifndef DFLTCC_UTIL_H +#define DFLTCC_UTIL_H + +#include <linux/zutil.h> + +/* + * C wrapper for the DEFLATE CONVERSION CALL instruction. + */ +typedef enum { + DFLTCC_CC_OK = 0, + DFLTCC_CC_OP1_TOO_SHORT = 1, + DFLTCC_CC_OP2_TOO_SHORT = 2, + DFLTCC_CC_OP2_CORRUPT = 2, + DFLTCC_CC_AGAIN = 3, +} dfltcc_cc; + +#define DFLTCC_QAF 0 +#define DFLTCC_GDHT 1 +#define DFLTCC_CMPR 2 +#define DFLTCC_XPND 4 +#define HBT_CIRCULAR (1 << 7) +#define HB_BITS 15 +#define HB_SIZE (1 << HB_BITS) + +static inline dfltcc_cc dfltcc( + int fn, + void *param, + Byte **op1, + size_t *len1, + const Byte **op2, + size_t *len2, + void *hist +) +{ + Byte *t2 = op1 ? *op1 : NULL; + size_t t3 = len1 ? *len1 : 0; + const Byte *t4 = op2 ? *op2 : NULL; + size_t t5 = len2 ? *len2 : 0; + register int r0 __asm__("r0") = fn; + register void *r1 __asm__("r1") = param; + register Byte *r2 __asm__("r2") = t2; + register size_t r3 __asm__("r3") = t3; + register const Byte *r4 __asm__("r4") = t4; + register size_t r5 __asm__("r5") = t5; + int cc; + + __asm__ volatile( + ".insn rrf,0xb9390000,%[r2],%[r4],%[hist],0\n" + "ipm %[cc]\n" + : [r2] "+r" (r2) + , [r3] "+r" (r3) + , [r4] "+r" (r4) + , [r5] "+r" (r5) + , [cc] "=r" (cc) + : [r0] "r" (r0) + , [r1] "r" (r1) + , [hist] "r" (hist) + : "cc", "memory"); + t2 = r2; t3 = r3; t4 = r4; t5 = r5; + + if (op1) + *op1 = t2; + if (len1) + *len1 = t3; + if (op2) + *op2 = t4; + if (len2) + *len2 = t5; + return (cc >> 28) & 3; +} + +static inline int is_bit_set( + const char *bits, + int n +) +{ + return bits[n / 8] & (1 << (7 - (n % 8))); +} + +static inline void turn_bit_off( + char *bits, + int n +) +{ + bits[n / 8] &= ~(1 << (7 - (n % 8))); +} + +static inline int dfltcc_are_params_ok( + int level, + uInt window_bits, + int strategy, + uLong level_mask +) +{ + return (level_mask & (1 << level)) != 0 && + (window_bits == HB_BITS) && + (strategy == Z_DEFAULT_STRATEGY); +} + +char *oesc_msg(char *buf, int oesc); + +#endif /* DFLTCC_UTIL_H */ diff --git a/lib/zlib_inflate/inffast.c b/lib/zlib_inflate/inffast.c index 2c13ecc5bb2c..2843f9bb42ac 100644 --- a/lib/zlib_inflate/inffast.c +++ b/lib/zlib_inflate/inffast.c @@ -10,23 +10,12 @@ #ifndef ASMINF -/* Allow machine dependent optimization for post-increment or pre-increment. - Based on testing to date, - Pre-increment preferred for: - - PowerPC G3 (Adler) - - MIPS R5000 (Randers-Pehrson) - Post-increment preferred for: - - none - No measurable difference: - - Pentium III (Anderson) - - M68060 (Nikl) - */ union uu { unsigned short us; unsigned char b[2]; }; -/* Endian independed version */ +/* Endian independent version */ static inline unsigned short get_unaligned16(const unsigned short *p) { @@ -38,16 +27,6 @@ get_unaligned16(const unsigned short *p) return mm.us; } -#ifdef POSTINC -# define OFF 0 -# define PUP(a) *(a)++ -# define UP_UNALIGNED(a) get_unaligned16((a)++) -#else -# define OFF 1 -# define PUP(a) *++(a) -# define UP_UNALIGNED(a) get_unaligned16(++(a)) -#endif - /* Decode literal, length, and distance codes and write out the resulting literal and match bytes until either not enough input or output is @@ -115,9 +94,9 @@ void inflate_fast(z_streamp strm, unsigned start) /* copy state to local variables */ state = (struct inflate_state *)strm->state; - in = strm->next_in - OFF; + in = strm->next_in; last = in + (strm->avail_in - 5); - out = strm->next_out - OFF; + out = strm->next_out; beg = out - (start - strm->avail_out); end = out + (strm->avail_out - 257); #ifdef INFLATE_STRICT @@ -138,9 +117,9 @@ void inflate_fast(z_streamp strm, unsigned start) input data or output space */ do { if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } this = lcode[hold & lmask]; @@ -150,14 +129,14 @@ void inflate_fast(z_streamp strm, unsigned start) bits -= op; op = (unsigned)(this.op); if (op == 0) { /* literal */ - PUP(out) = (unsigned char)(this.val); + *out++ = (unsigned char)(this.val); } else if (op & 16) { /* length base */ len = (unsigned)(this.val); op &= 15; /* number of extra bits */ if (op) { if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } len += (unsigned)hold & ((1U << op) - 1); @@ -165,9 +144,9 @@ void inflate_fast(z_streamp strm, unsigned start) bits -= op; } if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } this = dcode[hold & dmask]; @@ -180,10 +159,10 @@ void inflate_fast(z_streamp strm, unsigned start) dist = (unsigned)(this.val); op &= 15; /* number of extra bits */ if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } } @@ -205,13 +184,13 @@ void inflate_fast(z_streamp strm, unsigned start) state->mode = BAD; break; } - from = window - OFF; + from = window; if (write == 0) { /* very common case */ from += wsize - op; if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -222,14 +201,14 @@ void inflate_fast(z_streamp strm, unsigned start) if (op < len) { /* some from end of window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); - from = window - OFF; + from = window; if (write < len) { /* some from start of window */ op = write; len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -240,21 +219,21 @@ void inflate_fast(z_streamp strm, unsigned start) if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } } while (len > 2) { - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } else { @@ -264,29 +243,28 @@ void inflate_fast(z_streamp strm, unsigned start) from = out - dist; /* copy direct from output */ /* minimum length is three */ /* Align out addr */ - if (!((long)(out - 1 + OFF) & 1)) { - PUP(out) = PUP(from); + if (!((long)(out - 1) & 1)) { + *out++ = *from++; len--; } - sout = (unsigned short *)(out - OFF); + sout = (unsigned short *)(out); if (dist > 2) { unsigned short *sfrom; - sfrom = (unsigned short *)(from - OFF); + sfrom = (unsigned short *)(from); loops = len >> 1; - do -#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS - PUP(sout) = PUP(sfrom); -#else - PUP(sout) = UP_UNALIGNED(sfrom); -#endif - while (--loops); - out = (unsigned char *)sout + OFF; - from = (unsigned char *)sfrom + OFF; + do { + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) + *sout++ = *sfrom++; + else + *sout++ = get_unaligned16(sfrom++); + } while (--loops); + out = (unsigned char *)sout; + from = (unsigned char *)sfrom; } else { /* dist == 1 or dist == 2 */ unsigned short pat16; - pat16 = *(sout-1+OFF); + pat16 = *(sout-1); if (dist == 1) { union uu mm; /* copy one char pattern to both bytes */ @@ -296,12 +274,12 @@ void inflate_fast(z_streamp strm, unsigned start) } loops = len >> 1; do - PUP(sout) = pat16; + *sout++ = pat16; while (--loops); - out = (unsigned char *)sout + OFF; + out = (unsigned char *)sout; } if (len & 1) - PUP(out) = PUP(from); + *out++ = *from++; } } else if ((op & 64) == 0) { /* 2nd level distance code */ @@ -336,8 +314,8 @@ void inflate_fast(z_streamp strm, unsigned start) hold &= (1U << bits) - 1; /* update state and return */ - strm->next_in = in + OFF; - strm->next_out = out + OFF; + strm->next_in = in; + strm->next_out = out; strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); strm->avail_out = (unsigned)(out < end ? 257 + (end - out) : 257 - (out - end)); diff --git a/lib/zlib_inflate/inflate.c b/lib/zlib_inflate/inflate.c index 48f14cd58c77..ee39b5eb71f7 100644 --- a/lib/zlib_inflate/inflate.c +++ b/lib/zlib_inflate/inflate.c @@ -15,6 +15,16 @@ #include "inffast.h" #include "infutil.h" +/* architecture-specific bits */ +#ifdef CONFIG_ZLIB_DFLTCC +# include "../zlib_dfltcc/dfltcc.h" +#else +#define INFLATE_RESET_HOOK(strm) do {} while (0) +#define INFLATE_TYPEDO_HOOK(strm, flush) do {} while (0) +#define INFLATE_NEED_UPDATEWINDOW(strm) 1 +#define INFLATE_NEED_CHECKSUM(strm) 1 +#endif + int zlib_inflate_workspacesize(void) { return sizeof(struct inflate_workspace); @@ -42,6 +52,7 @@ int zlib_inflateReset(z_streamp strm) state->write = 0; state->whave = 0; + INFLATE_RESET_HOOK(strm); return Z_OK; } @@ -66,7 +77,15 @@ int zlib_inflateInit2(z_streamp strm, int windowBits) return Z_STREAM_ERROR; } state->wbits = (unsigned)windowBits; +#ifdef CONFIG_ZLIB_DFLTCC + /* + * DFLTCC requires the window to be page aligned. + * Thus, we overallocate and take the aligned portion of the buffer. + */ + state->window = PTR_ALIGN(&WS(strm)->working_window[0], PAGE_SIZE); +#else state->window = &WS(strm)->working_window[0]; +#endif return zlib_inflateReset(strm); } @@ -227,11 +246,6 @@ static int zlib_inflateSyncPacket(z_streamp strm) bits -= bits & 7; \ } while (0) -/* Reverse the bytes in a 32-bit value */ -#define REVERSE(q) \ - ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ - (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) - /* inflate() uses a state machine to process as much input data and generate as much output data as possible before returning. The state machine is @@ -382,7 +396,7 @@ int zlib_inflate(z_streamp strm, int flush) strm->adler = state->check = REVERSE(hold); INITBITS(); state->mode = DICT; - /* fall through */ + fallthrough; case DICT: if (state->havedict == 0) { RESTORE(); @@ -390,11 +404,12 @@ int zlib_inflate(z_streamp strm, int flush) } strm->adler = state->check = zlib_adler32(0L, NULL, 0); state->mode = TYPE; - /* fall through */ + fallthrough; case TYPE: if (flush == Z_BLOCK) goto inf_leave; - /* fall through */ + fallthrough; case TYPEDO: + INFLATE_TYPEDO_HOOK(strm, flush); if (state->last) { BYTEBITS(); state->mode = CHECK; @@ -431,7 +446,7 @@ int zlib_inflate(z_streamp strm, int flush) state->length = (unsigned)hold & 0xffff; INITBITS(); state->mode = COPY; - /* fall through */ + fallthrough; case COPY: copy = state->length; if (copy) { @@ -465,7 +480,7 @@ int zlib_inflate(z_streamp strm, int flush) #endif state->have = 0; state->mode = LENLENS; - /* fall through */ + fallthrough; case LENLENS: while (state->have < state->ncode) { NEEDBITS(3); @@ -486,7 +501,7 @@ int zlib_inflate(z_streamp strm, int flush) } state->have = 0; state->mode = CODELENS; - /* fall through */ + fallthrough; case CODELENS: while (state->have < state->nlen + state->ndist) { for (;;) { @@ -560,7 +575,7 @@ int zlib_inflate(z_streamp strm, int flush) break; } state->mode = LEN; - /* fall through */ + fallthrough; case LEN: if (have >= 6 && left >= 258) { RESTORE(); @@ -600,7 +615,7 @@ int zlib_inflate(z_streamp strm, int flush) } state->extra = (unsigned)(this.op) & 15; state->mode = LENEXT; - /* fall through */ + fallthrough; case LENEXT: if (state->extra) { NEEDBITS(state->extra); @@ -608,7 +623,7 @@ int zlib_inflate(z_streamp strm, int flush) DROPBITS(state->extra); } state->mode = DIST; - /* fall through */ + fallthrough; case DIST: for (;;) { this = state->distcode[BITS(state->distbits)]; @@ -634,7 +649,7 @@ int zlib_inflate(z_streamp strm, int flush) state->offset = (unsigned)this.val; state->extra = (unsigned)(this.op) & 15; state->mode = DISTEXT; - /* fall through */ + fallthrough; case DISTEXT: if (state->extra) { NEEDBITS(state->extra); @@ -654,7 +669,7 @@ int zlib_inflate(z_streamp strm, int flush) break; } state->mode = MATCH; - /* fall through */ + fallthrough; case MATCH: if (left == 0) goto inf_leave; copy = out - left; @@ -692,7 +707,7 @@ int zlib_inflate(z_streamp strm, int flush) out -= left; strm->total_out += out; state->total += out; - if (out) + if (INFLATE_NEED_CHECKSUM(strm) && out) strm->adler = state->check = UPDATE(state->check, put - out, out); out = left; @@ -705,7 +720,7 @@ int zlib_inflate(z_streamp strm, int flush) INITBITS(); } state->mode = DONE; - /* fall through */ + fallthrough; case DONE: ret = Z_STREAM_END; goto inf_leave; @@ -726,7 +741,8 @@ int zlib_inflate(z_streamp strm, int flush) */ inf_leave: RESTORE(); - if (state->wsize || (state->mode < CHECK && out != strm->avail_out)) + if (INFLATE_NEED_UPDATEWINDOW(strm) && + (state->wsize || (state->mode < CHECK && out != strm->avail_out))) zlib_updatewindow(strm, out); in -= strm->avail_in; @@ -734,7 +750,7 @@ int zlib_inflate(z_streamp strm, int flush) strm->total_in += in; strm->total_out += out; state->total += out; - if (state->wrap && out) + if (INFLATE_NEED_CHECKSUM(strm) && state->wrap && out) strm->adler = state->check = UPDATE(state->check, strm->next_out - out, out); diff --git a/lib/zlib_inflate/inflate.h b/lib/zlib_inflate/inflate.h index 3d17b3d1b21f..f79337ddf98c 100644 --- a/lib/zlib_inflate/inflate.h +++ b/lib/zlib_inflate/inflate.h @@ -11,6 +11,8 @@ subject to change. Applications should only use zlib.h. */ +#include "inftrees.h" + /* Possible inflate modes between inflate() calls */ typedef enum { HEAD, /* i: waiting for magic header */ @@ -108,4 +110,10 @@ struct inflate_state { unsigned short work[288]; /* work area for code table building */ code codes[ENOUGH]; /* space for code tables */ }; + +/* Reverse the bytes in a 32-bit value */ +#define REVERSE(q) \ + ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ + (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) + #endif diff --git a/lib/zlib_inflate/infutil.h b/lib/zlib_inflate/infutil.h index eb1a9007bd86..784ab33b7842 100644 --- a/lib/zlib_inflate/infutil.h +++ b/lib/zlib_inflate/infutil.h @@ -12,14 +12,28 @@ #define _INFUTIL_H #include <linux/zlib.h> +#ifdef CONFIG_ZLIB_DFLTCC +#include "../zlib_dfltcc/dfltcc.h" +#include <asm/page.h> +#endif /* memory allocation for inflation */ struct inflate_workspace { struct inflate_state inflate_state; - unsigned char working_window[1 << MAX_WBITS]; +#ifdef CONFIG_ZLIB_DFLTCC + struct dfltcc_state dfltcc_state; + unsigned char working_window[(1 << MAX_WBITS) + PAGE_SIZE]; +#else + unsigned char working_window[(1 << MAX_WBITS)]; +#endif }; -#define WS(z) ((struct inflate_workspace *)(z->workspace)) +#ifdef CONFIG_ZLIB_DFLTCC +/* dfltcc_state must be doubleword aligned for DFLTCC call */ +static_assert(offsetof(struct inflate_workspace, dfltcc_state) % 8 == 0); +#endif + +#define WS(strm) ((struct inflate_workspace *)(strm->workspace)) #endif diff --git a/lib/zstd/bitstream.h b/lib/zstd/bitstream.h index 3a49784d5c61..5d6343c1a909 100644 --- a/lib/zstd/bitstream.h +++ b/lib/zstd/bitstream.h @@ -259,16 +259,17 @@ ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, s bitD->bitContainer = *(const BYTE *)(bitD->start); switch (srcSize) { case 7: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16); - /* fall through */ + fallthrough; case 6: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24); - /* fall through */ + fallthrough; case 5: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32); - /* fall through */ + fallthrough; case 4: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[3]) << 24; - /* fall through */ + fallthrough; case 3: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[2]) << 16; - /* fall through */ + fallthrough; case 2: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[1]) << 8; + fallthrough; default:; } { diff --git a/lib/zstd/compress.c b/lib/zstd/compress.c index 5e0b67003e55..b080264ed3ad 100644 --- a/lib/zstd/compress.c +++ b/lib/zstd/compress.c @@ -3182,7 +3182,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream *zcs, void *dst, size_t * zcs->outBuffFlushedSize = 0; zcs->stage = zcss_flush; /* pass-through to flush stage */ } - /* fall through */ + fallthrough; case zcss_flush: { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; diff --git a/lib/zstd/decompress.c b/lib/zstd/decompress.c index 269ee9a796c1..66cd487a326a 100644 --- a/lib/zstd/decompress.c +++ b/lib/zstd/decompress.c @@ -442,7 +442,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx *dctx, const void *src, size_t srcSize case set_repeat: if (dctx->litEntropy == 0) return ERROR(dictionary_corrupted); - /* fall-through */ + fallthrough; case set_compressed: if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ @@ -1768,7 +1768,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, c return 0; } dctx->expected = 0; /* not necessary to copy more */ - /* fall through */ + fallthrough; case ZSTDds_decodeFrameHeader: memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); @@ -2309,7 +2309,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inB switch (zds->stage) { case zdss_init: ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ - /* fall-through */ + fallthrough; case zdss_loadHeader: { size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); @@ -2376,7 +2376,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inB } zds->stage = zdss_read; } - /* fall through */ + fallthrough; case zdss_read: { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); @@ -2405,7 +2405,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inB zds->stage = zdss_load; /* pass-through */ } - /* fall through */ + fallthrough; case zdss_load: { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); @@ -2438,7 +2438,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inB /* pass-through */ } } - /* fall through */ + fallthrough; case zdss_flush: { size_t const toFlushSize = zds->outEnd - zds->outStart; diff --git a/lib/zstd/fse_decompress.c b/lib/zstd/fse_decompress.c index a84300e5a013..0b353530fb3f 100644 --- a/lib/zstd/fse_decompress.c +++ b/lib/zstd/fse_decompress.c @@ -47,6 +47,7 @@ ****************************************************************/ #include "bitstream.h" #include "fse.h" +#include "zstd_internal.h" #include <linux/compiler.h> #include <linux/kernel.h> #include <linux/string.h> /* memcpy, memset */ @@ -60,14 +61,6 @@ enum { FSE_static_assert = 1 / (int)(!!(c)) }; \ } /* use only *after* variable declarations */ -/* check and forward error code */ -#define CHECK_F(f) \ - { \ - size_t const e = f; \ - if (FSE_isError(e)) \ - return e; \ - } - /* ************************************************************** * Templates ****************************************************************/ diff --git a/lib/zstd/huf.h b/lib/zstd/huf.h index 2143da28d952..923218d12e28 100644 --- a/lib/zstd/huf.h +++ b/lib/zstd/huf.h @@ -134,7 +134,7 @@ typedef enum { HUF_repeat_none, /**< Cannot use the previous table */ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ - HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } HUF_repeat; /** HUF_compress4X_repeat() : * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. diff --git a/lib/zstd/huf_compress.c b/lib/zstd/huf_compress.c index e727812d12aa..fd32838c185f 100644 --- a/lib/zstd/huf_compress.c +++ b/lib/zstd/huf_compress.c @@ -556,10 +556,11 @@ size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, si n = srcSize & ~3; /* join to mod 4 */ switch (srcSize & 3) { case 3: HUF_encodeSymbol(&bitC, ip[n + 2], CTable); HUF_FLUSHBITS_2(&bitC); - /* fall through */ + fallthrough; case 2: HUF_encodeSymbol(&bitC, ip[n + 1], CTable); HUF_FLUSHBITS_1(&bitC); - /* fall through */ + fallthrough; case 1: HUF_encodeSymbol(&bitC, ip[n + 0], CTable); HUF_FLUSHBITS(&bitC); + fallthrough; case 0: default:; } diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h index 1a79fab9e13a..dac753397f86 100644 --- a/lib/zstd/zstd_internal.h +++ b/lib/zstd/zstd_internal.h @@ -127,7 +127,14 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; * Shared functions to include for inlining *********************************************/ ZSTD_STATIC void ZSTD_copy8(void *dst, const void *src) { - memcpy(dst, src, 8); + /* + * zstd relies heavily on gcc being able to analyze and inline this + * memcpy() call, since it is called in a tight loop. Preboot mode + * is compiled in freestanding mode, which stops gcc from analyzing + * memcpy(). Use __builtin_memcpy() to tell gcc to analyze this as a + * regular memcpy(). + */ + __builtin_memcpy(dst, src, 8); } /*! ZSTD_wildcopy() : * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ @@ -137,13 +144,16 @@ ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length) const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; BYTE* const oend = op + length; - /* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388. +#if defined(GCC_VERSION) && GCC_VERSION >= 70000 && GCC_VERSION < 70200 + /* + * Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388. * Avoid the bad case where the loop only runs once by handling the * special case separately. This doesn't trigger the bug because it * doesn't involve pointer/integer overflow. */ if (length <= 8) return ZSTD_copy8(dst, src); +#endif do { ZSTD_copy8(op, ip); op += 8; |