From 49a9ef76740206d52e7393f6fe25fc764de8df32 Mon Sep 17 00:00:00 2001 From: Vinay Varma Date: Tue, 11 Apr 2023 17:17:15 +0800 Subject: scripts: `make rust-analyzer` for out-of-tree modules Adds support for out-of-tree rust modules to use the `rust-analyzer` make target to generate the rust-project.json file. The change involves adding an optional parameter `external_src` to the `generate_rust_analyzer.py` which expects the path to the out-of-tree module's source directory. When this parameter is passed, I have chosen not to add the non-core modules (samples and drivers) into the result since these are not expected to be used in third party modules. Related changes are also made to the Makefile and rust/Makefile allowing the `rust-analyzer` target to be used for out-of-tree modules as well. Link: https://github.com/Rust-for-Linux/linux/pull/914 Link: https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2 Signed-off-by: Vinay Varma Link: https://lore.kernel.org/r/20230411091714.130525-1-varmavinaym@gmail.com Signed-off-by: Miguel Ojeda --- scripts/generate_rust_analyzer.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index 946e250c1b2a..848fa1ad92ba 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -6,10 +6,11 @@ import argparse import json import logging +import os import pathlib import sys -def generate_crates(srctree, objtree, sysroot_src): +def generate_crates(srctree, objtree, sysroot_src, external_src): # Generate the configuration list. cfg = [] with open(objtree / "include" / "generated" / "rustc_cfg") as fd: @@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src): [], is_proc_macro=True, ) - crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so" + crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so" append_crate( "build_error", @@ -95,19 +96,26 @@ def generate_crates(srctree, objtree, sysroot_src): "exclude_dirs": [], } + def is_root_crate(build_file, target): + try: + return f"{target}.o" in open(build_file).read() + except FileNotFoundError: + return False + # Then, the rest outside of `rust/`. # # We explicitly mention the top-level folders we want to cover. - for folder in ("samples", "drivers"): - for path in (srctree / folder).rglob("*.rs"): + extra_dirs = map(lambda dir: srctree / dir, ("samples", "drivers")) + if external_src is not None: + extra_dirs = [external_src] + for folder in extra_dirs: + for path in folder.rglob("*.rs"): logging.info("Checking %s", path) name = path.name.replace(".rs", "") # Skip those that are not crate roots. - try: - if f"{name}.o" not in open(path.parent / "Makefile").read(): - continue - except FileNotFoundError: + if not is_root_crate(path.parent / "Makefile", name) and \ + not is_root_crate(path.parent / "Kbuild", name): continue logging.info("Adding %s", name) @@ -126,6 +134,7 @@ def main(): parser.add_argument("srctree", type=pathlib.Path) parser.add_argument("objtree", type=pathlib.Path) parser.add_argument("sysroot_src", type=pathlib.Path) + parser.add_argument("exttree", type=pathlib.Path, nargs="?") args = parser.parse_args() logging.basicConfig( @@ -134,7 +143,7 @@ def main(): ) rust_project = { - "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src), + "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree), "sysroot_src": str(args.sysroot_src), } -- cgit From d824d2f98565e7c4cb1b862c230198fbe1a968be Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 16 Jun 2023 02:16:21 +0200 Subject: kbuild: rust_is_available: remove -v option The -v option is passed when this script is invoked from Makefile, but not when invoked from Kconfig. As you can see in scripts/Kconfig.include, the 'success' macro suppresses stdout and stderr anyway, so this script does not need to be quiet. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20230109061436.3146442-1-masahiroy@kernel.org [ Reworded prefix to match the others in the patch series. ] Reviewed-by: Martin Rodriguez Reboredo Link: https://lore.kernel.org/r/20230616001631.463536-2-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/rust_is_available.sh | 96 ++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 56 deletions(-) (limited to 'scripts') diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index aebbf1913970..f43a010eaf30 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -2,8 +2,6 @@ # SPDX-License-Identifier: GPL-2.0 # # Tests whether a suitable Rust toolchain is available. -# -# Pass `-v` for human output and more checks (as warnings). set -e @@ -23,21 +21,17 @@ get_canonical_version() # Check that the Rust compiler exists. if ! command -v "$RUSTC" >/dev/null; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust compiler '$RUSTC' could not be found." - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust compiler '$RUSTC' could not be found." + echo >&2 "***" exit 1 fi # Check that the Rust bindings generator exists. if ! command -v "$BINDGEN" >/dev/null; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust bindings generator '$BINDGEN' could not be found." - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' could not be found." + echo >&2 "***" exit 1 fi @@ -53,16 +47,14 @@ rust_compiler_min_version=$($min_tool_version rustc) rust_compiler_cversion=$(get_canonical_version $rust_compiler_version) rust_compiler_min_cversion=$(get_canonical_version $rust_compiler_min_version) if [ "$rust_compiler_cversion" -lt "$rust_compiler_min_cversion" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust compiler '$RUSTC' is too old." - echo >&2 "*** Your version: $rust_compiler_version" - echo >&2 "*** Minimum version: $rust_compiler_min_version" - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust compiler '$RUSTC' is too old." + echo >&2 "*** Your version: $rust_compiler_version" + echo >&2 "*** Minimum version: $rust_compiler_min_version" + echo >&2 "***" exit 1 fi -if [ "$1" = -v ] && [ "$rust_compiler_cversion" -gt "$rust_compiler_min_cversion" ]; then +if [ "$rust_compiler_cversion" -gt "$rust_compiler_min_cversion" ]; then echo >&2 "***" echo >&2 "*** Rust compiler '$RUSTC' is too new. This may or may not work." echo >&2 "*** Your version: $rust_compiler_version" @@ -82,16 +74,14 @@ rust_bindings_generator_min_version=$($min_tool_version bindgen) rust_bindings_generator_cversion=$(get_canonical_version $rust_bindings_generator_version) rust_bindings_generator_min_cversion=$(get_canonical_version $rust_bindings_generator_min_version) if [ "$rust_bindings_generator_cversion" -lt "$rust_bindings_generator_min_cversion" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust bindings generator '$BINDGEN' is too old." - echo >&2 "*** Your version: $rust_bindings_generator_version" - echo >&2 "*** Minimum version: $rust_bindings_generator_min_version" - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' is too old." + echo >&2 "*** Your version: $rust_bindings_generator_version" + echo >&2 "*** Minimum version: $rust_bindings_generator_min_version" + echo >&2 "***" exit 1 fi -if [ "$1" = -v ] && [ "$rust_bindings_generator_cversion" -gt "$rust_bindings_generator_min_cversion" ]; then +if [ "$rust_bindings_generator_cversion" -gt "$rust_bindings_generator_min_cversion" ]; then echo >&2 "***" echo >&2 "*** Rust bindings generator '$BINDGEN' is too new. This may or may not work." echo >&2 "*** Your version: $rust_bindings_generator_version" @@ -110,13 +100,11 @@ bindgen_libclang_min_version=$($min_tool_version llvm) bindgen_libclang_cversion=$(get_canonical_version $bindgen_libclang_version) bindgen_libclang_min_cversion=$(get_canonical_version $bindgen_libclang_min_version) if [ "$bindgen_libclang_cversion" -lt "$bindgen_libclang_min_cversion" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN') is too old." - echo >&2 "*** Your version: $bindgen_libclang_version" - echo >&2 "*** Minimum version: $bindgen_libclang_min_version" - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN') is too old." + echo >&2 "*** Your version: $bindgen_libclang_version" + echo >&2 "*** Minimum version: $bindgen_libclang_min_version" + echo >&2 "***" exit 1 fi @@ -125,21 +113,19 @@ fi # # In the future, we might be able to perform a full version check, see # https://github.com/rust-lang/rust-bindgen/issues/2138. -if [ "$1" = -v ]; then - cc_name=$($(dirname $0)/cc-version.sh "$CC" | cut -f1 -d' ') - if [ "$cc_name" = Clang ]; then - clang_version=$( \ - LC_ALL=C "$CC" --version 2>/dev/null \ - | sed -nE '1s:.*version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' - ) - if [ "$clang_version" != "$bindgen_libclang_version" ]; then - echo >&2 "***" - echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN')" - echo >&2 "*** version does not match Clang's. This may be a problem." - echo >&2 "*** libclang version: $bindgen_libclang_version" - echo >&2 "*** Clang version: $clang_version" - echo >&2 "***" - fi +cc_name=$($(dirname $0)/cc-version.sh "$CC" | cut -f1 -d' ') +if [ "$cc_name" = Clang ]; then + clang_version=$( \ + LC_ALL=C "$CC" --version 2>/dev/null \ + | sed -nE '1s:.*version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' + ) + if [ "$clang_version" != "$bindgen_libclang_version" ]; then + echo >&2 "***" + echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN')" + echo >&2 "*** version does not match Clang's. This may be a problem." + echo >&2 "*** libclang version: $bindgen_libclang_version" + echo >&2 "*** Clang version: $clang_version" + echo >&2 "***" fi fi @@ -150,11 +136,9 @@ rustc_sysroot=$("$RUSTC" $KRUSTFLAGS --print sysroot) rustc_src=${RUST_LIB_SRC:-"$rustc_sysroot/lib/rustlib/src/rust/library"} rustc_src_core="$rustc_src/core/src/lib.rs" if [ ! -e "$rustc_src_core" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Source code for the 'core' standard library could not be found" - echo >&2 "*** at '$rustc_src_core'." - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Source code for the 'core' standard library could not be found" + echo >&2 "*** at '$rustc_src_core'." + echo >&2 "***" exit 1 fi -- cgit From dee3a6b819c96fc8b1907577f585fd66f5c0fefe Mon Sep 17 00:00:00 2001 From: Russell Currey Date: Fri, 16 Jun 2023 02:16:22 +0200 Subject: kbuild: rust_is_available: fix version check when CC has multiple arguments rust_is_available.sh uses cc-version.sh to identify which C compiler is in use, as scripts/Kconfig.include does. cc-version.sh isn't designed to be able to handle multiple arguments in one variable, i.e. "ccache clang". Its invocation in rust_is_available.sh quotes "$CC", which makes $1 == "ccache clang" instead of the intended $1 == ccache & $2 == clang. cc-version.sh could also be changed to handle having "ccache clang" as one argument, but it only has the one consumer upstream, making it simpler to fix the caller here. Signed-off-by: Russell Currey Fixes: 78521f3399ab ("scripts: add `rust_is_available.sh`") Link: https://github.com/Rust-for-Linux/linux/pull/873 [ Reworded title prefix and reflow line to 75 columns. ] Reviewed-by: Martin Rodriguez Reboredo Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20230616001631.463536-3-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/rust_is_available.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index f43a010eaf30..0c9be438e4cd 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -113,10 +113,10 @@ fi # # In the future, we might be able to perform a full version check, see # https://github.com/rust-lang/rust-bindgen/issues/2138. -cc_name=$($(dirname $0)/cc-version.sh "$CC" | cut -f1 -d' ') +cc_name=$($(dirname $0)/cc-version.sh $CC | cut -f1 -d' ') if [ "$cc_name" = Clang ]; then clang_version=$( \ - LC_ALL=C "$CC" --version 2>/dev/null \ + LC_ALL=C $CC --version 2>/dev/null \ | sed -nE '1s:.*version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) if [ "$clang_version" != "$bindgen_libclang_version" ]; then -- cgit From aac284b1eb420c9317475cacdd21dd0ae3c3eda8 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 16 Jun 2023 02:16:24 +0200 Subject: kbuild: rust_is_available: print docs reference People trying out the Rust support in the kernel may get warnings and errors from `scripts/rust_is_available.sh` from the `rustavailable` target or the build step. Some of those users may be following the Quick Start guide, but others may not (likely those getting warnings from the build step instead of the target). While the messages are fairly clear on what the problem is, it may not be clear how to solve the particular issue, especially for those not aware of the documentation. We could add all sorts of details on the script for each one, but it is better to point users to the documentation instead, where it is easily readable in different formats. It also avoids duplication. Thus add a reference to the documentation whenever the script fails or there is at least a warning. Reviewed-by: Finn Behrens Reviewed-by: Masahiro Yamada Reviewed-by: Nathan Chancellor Reviewed-by: Martin Rodriguez Reboredo Link: https://lore.kernel.org/r/20230616001631.463536-5-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/rust_is_available.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'scripts') diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index 0c9be438e4cd..6b8131d5b547 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -19,6 +19,20 @@ get_canonical_version() echo $((100000 * $1 + 100 * $2 + $3)) } +# Print a reference to the Quick Start guide in the documentation. +print_docs_reference() +{ + echo >&2 "***" + echo >&2 "*** Please see Documentation/rust/quick-start.rst for details" + echo >&2 "*** on how to set up the Rust support." + echo >&2 "***" +} + +# If the script fails for any reason, or if there was any warning, then +# print a reference to the documentation on exit. +warning=0 +trap 'if [ $? -ne 0 ] || [ $warning -ne 0 ]; then print_docs_reference; fi' EXIT + # Check that the Rust compiler exists. if ! command -v "$RUSTC" >/dev/null; then echo >&2 "***" @@ -60,6 +74,7 @@ if [ "$rust_compiler_cversion" -gt "$rust_compiler_min_cversion" ]; then echo >&2 "*** Your version: $rust_compiler_version" echo >&2 "*** Expected version: $rust_compiler_min_version" echo >&2 "***" + warning=1 fi # Check that the Rust bindings generator is suitable. @@ -87,6 +102,7 @@ if [ "$rust_bindings_generator_cversion" -gt "$rust_bindings_generator_min_cvers echo >&2 "*** Your version: $rust_bindings_generator_version" echo >&2 "*** Expected version: $rust_bindings_generator_min_version" echo >&2 "***" + warning=1 fi # Check that the `libclang` used by the Rust bindings generator is suitable. @@ -126,6 +142,7 @@ if [ "$cc_name" = Clang ]; then echo >&2 "*** libclang version: $bindgen_libclang_version" echo >&2 "*** Clang version: $clang_version" echo >&2 "***" + warning=1 fi fi -- cgit From 52cae7f28ed6c3992489f16bb355f5b623f0912e Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 16 Jun 2023 02:16:25 +0200 Subject: kbuild: rust_is_available: add check for `bindgen` invocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `scripts/rust_is_available.sh` calls `bindgen` with a special header in order to check whether the `libclang` version in use is suitable. However, the invocation itself may fail if, for instance, `bindgen` cannot locate `libclang`. This is fine for Kconfig (since the script will still fail and therefore disable Rust as it should), but it is pretty confusing for users of the `rustavailable` target given the error will be unrelated: ./scripts/rust_is_available.sh: 21: arithmetic expression: expecting primary: "100000 * + 100 * + " make: *** [Makefile:1816: rustavailable] Error 2 Instead, run the `bindgen` invocation independently in a previous step, saving its output and return code. If it fails, then show the user a proper error message. Otherwise, continue as usual with the saved output. Since the previous patch we show a reference to the docs, and the docs now explain how `bindgen` looks for `libclang`, thus the error message can leverage the documentation, avoiding duplication here (and making users aware of the setup guide in the documentation). Reported-by: Nick Desaulniers Link: https://lore.kernel.org/rust-for-linux/CAKwvOdm5JT4wbdQQYuW+RT07rCi6whGBM2iUAyg8A1CmLXG6Nw@mail.gmail.com/ Reported-by: François Valenduc Closes: https://github.com/Rust-for-Linux/linux/issues/934 Reported-by: Alexandru Radovici Closes: https://github.com/Rust-for-Linux/linux/pull/921 Reported-by: Matthew Leach Closes: https://lore.kernel.org/rust-for-linux/20230507084116.1099067-1-dev@mattleach.net/ Fixes: 78521f3399ab ("scripts: add `rust_is_available.sh`") Reviewed-by: Martin Rodriguez Reboredo Reviewed-by: Masahiro Yamada Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20230616001631.463536-6-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/rust_is_available.sh | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index 6b8131d5b547..1bdff4472cbe 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -106,8 +106,28 @@ if [ "$rust_bindings_generator_cversion" -gt "$rust_bindings_generator_min_cvers fi # Check that the `libclang` used by the Rust bindings generator is suitable. +# +# In order to do that, first invoke `bindgen` to get the `libclang` version +# found by `bindgen`. This step may already fail if, for instance, `libclang` +# is not found, thus inform the user in such a case. +bindgen_libclang_output=$( \ + LC_ALL=C "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang.h 2>&1 >/dev/null +) || bindgen_libclang_code=$? +if [ -n "$bindgen_libclang_code" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the libclang version (used by the Rust" + echo >&2 "*** bindings generator) failed with code $bindgen_libclang_code. This may be caused by" + echo >&2 "*** a failure to locate libclang. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$bindgen_libclang_output" + echo >&2 "***" + exit 1 +fi + +# `bindgen` returned successfully, thus use the output to check that the version +# of the `libclang` found by the Rust bindings generator is suitable. bindgen_libclang_version=$( \ - LC_ALL=C "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang.h 2>&1 >/dev/null \ + echo "$bindgen_libclang_output" \ | grep -F 'clang version ' \ | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ | head -n 1 \ -- cgit From e90db5521de2e00b63ba425b3b215f02563efe0a Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 16 Jun 2023 02:16:26 +0200 Subject: kbuild: rust_is_available: check that environment variables are set Sometimes [1] users may attempt to setup the Rust support by checking what Kbuild does and they end up finding out about `scripts/rust_is_available.sh`. Inevitably, they run the script directly, but unless they setup the required variables, the result of the script is not meaningful. We could add some defaults to the variables, but that could be confusing for those that may override the defaults (compared to their kernel builds), and `$CC` would not be a simple default in any case. Therefore, instead, explicitly check whether the expected variables are set (`$RUSTC`, `$BINDGEN` and `$CC`). If not, print an explanation about the fact that the script is meant to be called from Kbuild, since that is the most likely cause for the variables not being set. Link: https://lore.kernel.org/oe-kbuild-all/Y6r4mXz5NS0+HVXo@zn.tnic/ [1] Reviewed-by: Martin Rodriguez Reboredo Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20230616001631.463536-7-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/rust_is_available.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'scripts') diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index 1bdff4472cbe..7e0368babe64 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -28,11 +28,40 @@ print_docs_reference() echo >&2 "***" } +# Print an explanation about the fact that the script is meant to be called from Kbuild. +print_kbuild_explanation() +{ + echo >&2 "***" + echo >&2 "*** This script is intended to be called from Kbuild." + echo >&2 "*** Please use the 'rustavailable' target to call it instead." + echo >&2 "*** Otherwise, the results may not be meaningful." + exit 1 +} + # If the script fails for any reason, or if there was any warning, then # print a reference to the documentation on exit. warning=0 trap 'if [ $? -ne 0 ] || [ $warning -ne 0 ]; then print_docs_reference; fi' EXIT +# Check that the expected environment variables are set. +if [ -z "${RUSTC+x}" ]; then + echo >&2 "***" + echo >&2 "*** Environment variable 'RUSTC' is not set." + print_kbuild_explanation +fi + +if [ -z "${BINDGEN+x}" ]; then + echo >&2 "***" + echo >&2 "*** Environment variable 'BINDGEN' is not set." + print_kbuild_explanation +fi + +if [ -z "${CC+x}" ]; then + echo >&2 "***" + echo >&2 "*** Environment variable 'CC' is not set." + print_kbuild_explanation +fi + # Check that the Rust compiler exists. if ! command -v "$RUSTC" >/dev/null; then echo >&2 "***" -- cgit From 9eb7e20e0c5cd069457845f965b3e8a7d736ecb7 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 16 Jun 2023 02:16:27 +0200 Subject: kbuild: rust_is_available: fix confusion when a version appears in the path `bindgen`'s output for `libclang`'s version check contains paths, which in turn may contain strings that look like version numbers [1][2]: .../6.1.0-dev/.../rust_is_available_bindgen_libclang.h:2:9: warning: clang version 11.1.0 [-W#pragma-messages], err: false which the script will pick up as the version instead of the latter. It is also the case that versions may appear after the actual version (e.g. distribution's version text), which was the reason behind `head` [3]: .../rust-is-available-bindgen-libclang.h:2:9: warning: clang version 13.0.0 (Fedora 13.0.0-3.fc35) [-W#pragma-messages], err: false Thus instead ask for a match after the `clang version` string. Reported-by: Jordan Isaacs Closes: https://github.com/Rust-for-Linux/linux/issues/942 [1] Reported-by: "Ethan D. Twardy" Closes: https://lore.kernel.org/rust-for-linux/20230528131802.6390-2-ethan.twardy@gmail.com/ [2] Reported-by: Tiago Lam Closes: https://github.com/Rust-for-Linux/linux/pull/789 [3] Fixes: 78521f3399ab ("scripts: add `rust_is_available.sh`") Reviewed-by: Martin Rodriguez Reboredo Reviewed-by: Ethan Twardy Tested-by: Ethan Twardy Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20230616001631.463536-8-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/rust_is_available.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index 7e0368babe64..810691af66eb 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -157,9 +157,7 @@ fi # of the `libclang` found by the Rust bindings generator is suitable. bindgen_libclang_version=$( \ echo "$bindgen_libclang_output" \ - | grep -F 'clang version ' \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ - | head -n 1 \ + | sed -nE 's:.*clang version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) bindgen_libclang_min_version=$($min_tool_version llvm) bindgen_libclang_cversion=$(get_canonical_version $bindgen_libclang_version) -- cgit From 7cd6a3e1f94bab4f2a3425e06f70ab13eb8190d4 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 16 Jun 2023 02:16:28 +0200 Subject: kbuild: rust_is_available: normalize version matching In order to match the version string, `sed` is used in a couple cases, and `grep` and `head` in a couple others. Make the script more consistent and easier to understand by using the same method, `sed`, for all of them. This makes the version matching also a bit more strict for the changed cases, since the strings `rustc ` and `bindgen ` will now be required, which should be fine since `rustc` complains if one attempts to call it with another program name, and `bindgen` uses a hardcoded string. In addition, clarify why one of the existing `sed` commands does not provide an address like the others. Reviewed-by: Nathan Chancellor Reviewed-by: Masahiro Yamada Reviewed-by: Martin Rodriguez Reboredo Link: https://lore.kernel.org/r/20230616001631.463536-9-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/rust_is_available.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index 810691af66eb..b7e0781fdea9 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -83,8 +83,7 @@ fi # Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. rust_compiler_version=$( \ LC_ALL=C "$RUSTC" --version 2>/dev/null \ - | head -n 1 \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ + | sed -nE '1s:.*rustc ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) rust_compiler_min_version=$($min_tool_version rustc) rust_compiler_cversion=$(get_canonical_version $rust_compiler_version) @@ -111,8 +110,7 @@ fi # Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. rust_bindings_generator_version=$( \ LC_ALL=C "$BINDGEN" --version 2>/dev/null \ - | head -n 1 \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ + | sed -nE '1s:.*bindgen ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) rust_bindings_generator_min_version=$($min_tool_version bindgen) rust_bindings_generator_cversion=$(get_canonical_version $rust_bindings_generator_version) @@ -155,6 +153,9 @@ fi # `bindgen` returned successfully, thus use the output to check that the version # of the `libclang` found by the Rust bindings generator is suitable. +# +# Unlike other version checks, note that this one does not necessarily appear +# in the first line of the output, thus no `sed` address is provided. bindgen_libclang_version=$( \ echo "$bindgen_libclang_output" \ | sed -nE 's:.*clang version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' -- cgit From f295522886a4ebb628cadb2cd74d0661d6292978 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 16 Jun 2023 02:16:29 +0200 Subject: kbuild: rust_is_available: handle failures calling `$RUSTC`/`$BINDGEN` The script already checks if `$RUSTC` and `$BINDGEN` exists via `command`, but the environment variables may point to a non-executable file, or the programs may fail for some other reason. While the script successfully exits with a failure as it should, the error given can be quite confusing depending on the shell and the behavior of its `command`. For instance, with `dash`: $ RUSTC=./mm BINDGEN=bindgen CC=clang scripts/rust_is_available.sh scripts/rust_is_available.sh: 19: arithmetic expression: expecting primary: "100000 * + 100 * + " Thus detect failure exit codes when calling `$RUSTC` and `$BINDGEN` and print a better message, in a similar way to what we do when extracting the `libclang` version found by `bindgen`. Link: https://lore.kernel.org/rust-for-linux/CAK7LNAQYk6s11MASRHW6oxtkqF00EJVqhHOP=5rynWt-QDUsXw@mail.gmail.com/ Reviewed-by: Nathan Chancellor Reviewed-by: Martin Rodriguez Reboredo Link: https://lore.kernel.org/r/20230616001631.463536-10-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/rust_is_available.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index b7e0781fdea9..da8296cd9b8d 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -81,8 +81,20 @@ fi # Check that the Rust compiler version is suitable. # # Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. +rust_compiler_output=$( \ + LC_ALL=C "$RUSTC" --version 2>/dev/null +) || rust_compiler_code=$? +if [ -n "$rust_compiler_code" ]; then + echo >&2 "***" + echo >&2 "*** Running '$RUSTC' to check the Rust compiler version failed with" + echo >&2 "*** code $rust_compiler_code. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_compiler_output" + echo >&2 "***" + exit 1 +fi rust_compiler_version=$( \ - LC_ALL=C "$RUSTC" --version 2>/dev/null \ + echo "$rust_compiler_output" \ | sed -nE '1s:.*rustc ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) rust_compiler_min_version=$($min_tool_version rustc) @@ -108,8 +120,20 @@ fi # Check that the Rust bindings generator is suitable. # # Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. +rust_bindings_generator_output=$( \ + LC_ALL=C "$BINDGEN" --version 2>/dev/null +) || rust_bindings_generator_code=$? +if [ -n "$rust_bindings_generator_code" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the Rust bindings generator version failed with" + echo >&2 "*** code $rust_bindings_generator_code. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_bindings_generator_output" + echo >&2 "***" + exit 1 +fi rust_bindings_generator_version=$( \ - LC_ALL=C "$BINDGEN" --version 2>/dev/null \ + echo "$rust_bindings_generator_output" \ | sed -nE '1s:.*bindgen ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) rust_bindings_generator_min_version=$($min_tool_version bindgen) -- cgit From bc60c930a43c7c984c80e99282f0d4f7193f3986 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 16 Jun 2023 02:16:30 +0200 Subject: kbuild: rust_is_available: check that output looks as expected The script already checks for `$RUSTC` and `$BINDGEN` existing and exiting without failure. However, one may still pass an unexpected binary that does not output what the later parsing expects. The script still successfully reports a failure as expected, but the error is confusing. For instance: $ RUSTC=true BINDGEN=bindgen CC=clang scripts/rust_is_available.sh scripts/rust_is_available.sh: 19: arithmetic expression: expecting primary: "100000 * + 100 * + " *** *** Please see Documentation/rust/quick-start.rst for details *** on how to set up the Rust support. *** Thus add an explicit check and a proper message for unexpected output from the called command. Similarly, do so for the `libclang` version parsing, too. Link: https://lore.kernel.org/rust-for-linux/CAK7LNAQYk6s11MASRHW6oxtkqF00EJVqhHOP=5rynWt-QDUsXw@mail.gmail.com/ Reviewed-by: Nathan Chancellor Reviewed-by: Martin Rodriguez Reboredo Link: https://lore.kernel.org/r/20230616001631.463536-11-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/rust_is_available.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'scripts') diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index da8296cd9b8d..117018946b57 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -97,6 +97,15 @@ rust_compiler_version=$( \ echo "$rust_compiler_output" \ | sed -nE '1s:.*rustc ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) +if [ -z "$rust_compiler_version" ]; then + echo >&2 "***" + echo >&2 "*** Running '$RUSTC' to check the Rust compiler version did not return" + echo >&2 "*** an expected output. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_compiler_output" + echo >&2 "***" + exit 1 +fi rust_compiler_min_version=$($min_tool_version rustc) rust_compiler_cversion=$(get_canonical_version $rust_compiler_version) rust_compiler_min_cversion=$(get_canonical_version $rust_compiler_min_version) @@ -136,6 +145,15 @@ rust_bindings_generator_version=$( \ echo "$rust_bindings_generator_output" \ | sed -nE '1s:.*bindgen ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) +if [ -z "$rust_bindings_generator_version" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the bindings generator version did not return" + echo >&2 "*** an expected output. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_bindings_generator_output" + echo >&2 "***" + exit 1 +fi rust_bindings_generator_min_version=$($min_tool_version bindgen) rust_bindings_generator_cversion=$(get_canonical_version $rust_bindings_generator_version) rust_bindings_generator_min_cversion=$(get_canonical_version $rust_bindings_generator_min_version) @@ -184,6 +202,16 @@ bindgen_libclang_version=$( \ echo "$bindgen_libclang_output" \ | sed -nE 's:.*clang version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) +if [ -z "$bindgen_libclang_version" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the libclang version (used by the Rust" + echo >&2 "*** bindings generator) did not return an expected output. See output" + echo >&2 "*** and docs below for details:" + echo >&2 "***" + echo >&2 "$bindgen_libclang_output" + echo >&2 "***" + exit 1 +fi bindgen_libclang_min_version=$($min_tool_version llvm) bindgen_libclang_cversion=$(get_canonical_version $bindgen_libclang_version) bindgen_libclang_min_cversion=$(get_canonical_version $bindgen_libclang_min_version) -- cgit From 0bb1c9282e2cb38d199347d1d96b77f208b64810 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 16 Jun 2023 02:16:31 +0200 Subject: kbuild: rust_is_available: add test suite The `rust_is_available.sh` script runs for everybody compiling the kernel, even if not using Rust. Therefore, it is important to ensure that the script is correct to avoid breaking people's compilation. In addition, the script needs to be able to handle a set of subtle cases, including parsing version strings of different tools. Therefore, maintenance of this script can be greatly eased with a set of tests. Thus add a test suite to cover hopefully most of the setups that the script may encounter in the wild. Extra setups can be easily added later on if missing. The script currently covers all the branches of the shell script, including several ways in which they may be entered. Python is used for this script, since the script under test does not depend on Rust, thus hopefully making it easier for others to use if the need arises. Reviewed-by: Martin Rodriguez Reboredo Link: https://lore.kernel.org/r/20230616001631.463536-12-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/rust_is_available_test.py | 346 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 346 insertions(+) create mode 100755 scripts/rust_is_available_test.py (limited to 'scripts') diff --git a/scripts/rust_is_available_test.py b/scripts/rust_is_available_test.py new file mode 100755 index 000000000000..57613fe5ed75 --- /dev/null +++ b/scripts/rust_is_available_test.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +"""Tests the `rust_is_available.sh` script. + +Some of the tests require the real programs to be available in `$PATH` +under their canonical name (and with the expected versions). +""" + +import enum +import os +import pathlib +import stat +import subprocess +import tempfile +import unittest + +class TestRustIsAvailable(unittest.TestCase): + @enum.unique + class Expected(enum.Enum): + SUCCESS = enum.auto() + SUCCESS_WITH_WARNINGS = enum.auto() + SUCCESS_WITH_EXTRA_OUTPUT = enum.auto() + FAILURE = enum.auto() + + @classmethod + def generate_executable(cls, content): + path = pathlib.Path(cls.tempdir.name) + name = str(len(tuple(path.iterdir()))) + path = path / name + with open(path, "w") as file_: + file_.write(content) + os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR) + return path + + @classmethod + def generate_clang(cls, stdout): + return cls.generate_executable(f"""#!/usr/bin/env python3 +import sys +if "-E" in " ".join(sys.argv): + print({repr("Clang " + " ".join(cls.llvm_default_version.split(" ")))}) +else: + print({repr(stdout)}) +""") + + @classmethod + def generate_rustc(cls, stdout): + return cls.generate_executable(f"""#!/usr/bin/env python3 +import sys +if "--print sysroot" in " ".join(sys.argv): + print({repr(cls.rust_default_sysroot)}) +else: + print({repr(stdout)}) +""") + + @classmethod + def generate_bindgen(cls, version_stdout, libclang_stderr): + return cls.generate_executable(f"""#!/usr/bin/env python3 +import sys +if "rust_is_available_bindgen_libclang.h" in " ".join(sys.argv): + print({repr(libclang_stderr)}, file=sys.stderr) +else: + print({repr(version_stdout)}) +""") + + @classmethod + def generate_bindgen_version(cls, stdout): + return cls.generate_bindgen(stdout, cls.bindgen_default_bindgen_libclang_stderr) + + @classmethod + def generate_bindgen_libclang(cls, stderr): + return cls.generate_bindgen(cls.bindgen_default_bindgen_version_stdout, stderr) + + @classmethod + def setUpClass(cls): + cls.tempdir = tempfile.TemporaryDirectory() + + cls.missing = pathlib.Path(cls.tempdir.name) / "missing" + + cls.nonexecutable = pathlib.Path(cls.tempdir.name) / "nonexecutable" + with open(cls.nonexecutable, "w") as file_: + file_.write("nonexecutable") + + cls.unexpected_binary = "true" + + cls.rustc_default_version = subprocess.check_output(("scripts/min-tool-version.sh", "rustc")).decode().strip() + cls.bindgen_default_version = subprocess.check_output(("scripts/min-tool-version.sh", "bindgen")).decode().strip() + cls.llvm_default_version = subprocess.check_output(("scripts/min-tool-version.sh", "llvm")).decode().strip() + cls.rust_default_sysroot = subprocess.check_output(("rustc", "--print", "sysroot")).decode().strip() + + cls.bindgen_default_bindgen_version_stdout = f"bindgen {cls.bindgen_default_version}" + cls.bindgen_default_bindgen_libclang_stderr = f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {cls.llvm_default_version} [-W#pragma-messages], err: false" + + cls.default_rustc = cls.generate_rustc(f"rustc {cls.rustc_default_version}") + cls.default_bindgen = cls.generate_bindgen(cls.bindgen_default_bindgen_version_stdout, cls.bindgen_default_bindgen_libclang_stderr) + cls.default_cc = cls.generate_clang(f"clang version {cls.llvm_default_version}") + + def run_script(self, expected, override_env): + env = { + "RUSTC": self.default_rustc, + "BINDGEN": self.default_bindgen, + "CC": self.default_cc, + } + + for key, value in override_env.items(): + if value is None: + del env[key] + continue + env[key] = value + + result = subprocess.run("scripts/rust_is_available.sh", env=env, capture_output=True) + + # The script should never output anything to `stdout`. + self.assertEqual(result.stdout, b"") + + if expected == self.Expected.SUCCESS: + # When expecting a success, the script should return 0 + # and it should not output anything to `stderr`. + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stderr, b"") + elif expected == self.Expected.SUCCESS_WITH_EXTRA_OUTPUT: + # When expecting a success with extra output (that is not warnings, + # which is the common case), the script should return 0 and it + # should output at least something to `stderr` (the output should + # be checked further by the test). + self.assertEqual(result.returncode, 0) + self.assertNotEqual(result.stderr, b"") + elif expected == self.Expected.SUCCESS_WITH_WARNINGS: + # When expecting a success with warnings, the script should return 0 + # and it should output at least the instructions to `stderr`. + self.assertEqual(result.returncode, 0) + self.assertIn(b"Please see Documentation/rust/quick-start.rst for details", result.stderr) + else: + # When expecting a failure, the script should return non-0 + # and it should output at least the instructions to `stderr`. + self.assertNotEqual(result.returncode, 0) + self.assertIn(b"Please see Documentation/rust/quick-start.rst for details", result.stderr) + + # The output will generally be UTF-8 (i.e. unless the user has + # put strange values in the environment). + result.stderr = result.stderr.decode() + + return result + + def test_rustc_unset(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": None }) + self.assertIn("Environment variable 'RUSTC' is not set.", result.stderr) + self.assertIn("This script is intended to be called from Kbuild.", result.stderr) + + def test_bindgen_unset(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": None }) + self.assertIn("Environment variable 'BINDGEN' is not set.", result.stderr) + self.assertIn("This script is intended to be called from Kbuild.", result.stderr) + + def test_cc_unset(self): + result = self.run_script(self.Expected.FAILURE, { "CC": None }) + self.assertIn("Environment variable 'CC' is not set.", result.stderr) + self.assertIn("This script is intended to be called from Kbuild.", result.stderr) + + def test_rustc_missing(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.missing }) + self.assertIn(f"Rust compiler '{self.missing}' could not be found.", result.stderr) + + def test_bindgen_missing(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.missing }) + self.assertIn(f"Rust bindings generator '{self.missing}' could not be found.", result.stderr) + + def test_rustc_nonexecutable(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.nonexecutable }) + self.assertIn(f"Running '{self.nonexecutable}' to check the Rust compiler version failed with", result.stderr) + + def test_rustc_unexpected_binary(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.unexpected_binary }) + self.assertIn(f"Running '{self.unexpected_binary}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_unexpected_name(self): + rustc = self.generate_rustc(f"unexpected {self.rustc_default_version} (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Running '{rustc}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_unexpected_version(self): + rustc = self.generate_rustc("rustc unexpected (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Running '{rustc}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_no_minor(self): + rustc = self.generate_rustc(f"rustc {'.'.join(self.rustc_default_version.split('.')[:2])} (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Running '{rustc}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_old_version(self): + rustc = self.generate_rustc("rustc 1.60.0 (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Rust compiler '{rustc}' is too old.", result.stderr) + + def test_rustc_new_version(self): + rustc = self.generate_rustc("rustc 1.999.0 (a8314ef7d 2099-06-27)") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "RUSTC": rustc }) + self.assertIn(f"Rust compiler '{rustc}' is too new. This may or may not work.", result.stderr) + + def test_bindgen_nonexecutable(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.nonexecutable }) + self.assertIn(f"Running '{self.nonexecutable}' to check the Rust bindings generator version failed with", result.stderr) + + def test_bindgen_unexpected_binary(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.unexpected_binary }) + self.assertIn(f"Running '{self.unexpected_binary}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_unexpected_name(self): + bindgen = self.generate_bindgen_version(f"unexpected {self.bindgen_default_version}") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_unexpected_version(self): + bindgen = self.generate_bindgen_version("bindgen unexpected") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_no_minor(self): + bindgen = self.generate_bindgen_version(f"bindgen {'.'.join(self.bindgen_default_version.split('.')[:2])}") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_old_version(self): + bindgen = self.generate_bindgen_version("bindgen 0.50.0") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Rust bindings generator '{bindgen}' is too old.", result.stderr) + + def test_bindgen_new_version(self): + bindgen = self.generate_bindgen_version("bindgen 0.999.0") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) + self.assertIn(f"Rust bindings generator '{bindgen}' is too new. This may or may not work.", result.stderr) + + def test_bindgen_libclang_failure(self): + for env in ( + { "LLVM_CONFIG_PATH": self.missing }, + { "LIBCLANG_PATH": self.missing }, + { "CLANG_PATH": self.missing }, + ): + with self.subTest(env=env): + result = self.run_script(self.Expected.FAILURE, env | { "PATH": os.environ["PATH"], "BINDGEN": "bindgen" }) + self.assertIn("Running 'bindgen' to check the libclang version (used by the Rust", result.stderr) + self.assertIn("bindings generator) failed with code ", result.stderr) + + def test_bindgen_libclang_unexpected_version(self): + bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version unexpected [-W#pragma-messages], err: false") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the libclang version (used by the Rust", result.stderr) + self.assertIn("bindings generator) did not return an expected output. See output", result.stderr) + + def test_bindgen_libclang_old_version(self): + bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version 10.0.0 [-W#pragma-messages], err: false") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"libclang (used by the Rust bindings generator '{bindgen}') is too old.", result.stderr) + + def test_clang_matches_bindgen_libclang_different_bindgen(self): + bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version 999.0.0 [-W#pragma-messages], err: false") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) + self.assertIn("version does not match Clang's. This may be a problem.", result.stderr) + + def test_clang_matches_bindgen_libclang_different_clang(self): + cc = self.generate_clang("clang version 999.0.0") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "CC": cc }) + self.assertIn("version does not match Clang's. This may be a problem.", result.stderr) + + def test_rustc_src_core_krustflags(self): + result = self.run_script(self.Expected.FAILURE, { "PATH": os.environ["PATH"], "RUSTC": "rustc", "KRUSTFLAGS": f"--sysroot={self.missing}" }) + self.assertIn("Source code for the 'core' standard library could not be found", result.stderr) + + def test_rustc_src_core_rustlibsrc(self): + result = self.run_script(self.Expected.FAILURE, { "RUST_LIB_SRC": self.missing }) + self.assertIn("Source code for the 'core' standard library could not be found", result.stderr) + + def test_success_cc_unknown(self): + result = self.run_script(self.Expected.SUCCESS_WITH_EXTRA_OUTPUT, { "CC": self.missing }) + self.assertIn("unknown C compiler", result.stderr) + + def test_success_cc_multiple_arguments_ccache(self): + clang = self.generate_clang(f"""Ubuntu clang version {self.llvm_default_version}-1ubuntu1 +Target: x86_64-pc-linux-gnu +Thread model: posix +InstalledDir: /usr/bin +""") + result = self.run_script(self.Expected.SUCCESS, { "CC": f"{clang} clang" }) + + def test_success_rustc_version(self): + for rustc_stdout in ( + f"rustc {self.rustc_default_version} (a8314ef7d 2022-06-27)", + f"rustc {self.rustc_default_version}-dev (a8314ef7d 2022-06-27)", + f"rustc {self.rustc_default_version}-1.60.0 (a8314ef7d 2022-06-27)", + ): + with self.subTest(rustc_stdout=rustc_stdout): + rustc = self.generate_rustc(rustc_stdout) + result = self.run_script(self.Expected.SUCCESS, { "RUSTC": rustc }) + + def test_success_bindgen_version(self): + for bindgen_stdout in ( + f"bindgen {self.bindgen_default_version}", + f"bindgen {self.bindgen_default_version}-dev", + f"bindgen {self.bindgen_default_version}-0.999.0", + ): + with self.subTest(bindgen_stdout=bindgen_stdout): + bindgen = self.generate_bindgen_version(bindgen_stdout) + result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) + + def test_success_bindgen_libclang(self): + for stderr in ( + f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (https://github.com/llvm/llvm-project.git 4a2c05b05ed07f1f620e94f6524a8b4b2760a0b1) [-W#pragma-messages], err: false", + f"/home/jd/Documents/dev/kernel-module-flake/linux-6.1/outputs/dev/lib/modules/6.1.0-development/source/scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} [-W#pragma-messages], err: false", + f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (Fedora 13.0.0-3.fc35) [-W#pragma-messages], err: false", + f""" +/nix/store/dsd5gz46hdbdk2rfdimqddhq6m8m8fqs-bash-5.1-p16/bin/bash: warning: setlocale: LC_ALL: cannot change locale (c) +scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} [-W#pragma-messages], err: false +""", + f""" +/nix/store/dsd5gz46hdbdk2rfdimqddhq6m8m8fqs-bash-5.1.0-p16/bin/bash: warning: setlocale: LC_ALL: cannot change locale (c) +/home/jd/Documents/dev/kernel-module-flake/linux-6.1/outputs/dev/lib/modules/6.1.0-development/source/scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (Fedora 13.0.0-3.fc35) [-W#pragma-messages], err: false +""" + ): + with self.subTest(stderr=stderr): + bindgen = self.generate_bindgen_libclang(stderr) + result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) + + def test_success_clang_version(self): + for clang_stdout in ( + f"clang version {self.llvm_default_version} (https://github.com/llvm/llvm-project.git 4a2c05b05ed07f1f620e94f6524a8b4b2760a0b1)", + f"clang version {self.llvm_default_version}-dev", + f"clang version {self.llvm_default_version}-2~ubuntu20.04.1", + f"Ubuntu clang version {self.llvm_default_version}-2~ubuntu20.04.1", + ): + with self.subTest(clang_stdout=clang_stdout): + clang = self.generate_clang(clang_stdout) + result = self.run_script(self.Expected.SUCCESS, { "CC": clang }) + + def test_success_real_programs(self): + for cc in ["gcc", "clang"]: + with self.subTest(cc=cc): + result = self.run_script(self.Expected.SUCCESS, { + "PATH": os.environ["PATH"], + "RUSTC": "rustc", + "BINDGEN": "bindgen", + "CC": cc, + }) + +if __name__ == "__main__": + unittest.main() -- cgit From 41bdc6decda074afc4d8f8ba44c69b08d0e9aff6 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 4 Jul 2023 07:21:36 +0200 Subject: btf, scripts: rust: drop is_rust_module.sh With commit c1177979af9c ("btf, scripts: Exclude Rust CUs with pahole") we are now able to use pahole directly to identify Rust compilation units (CUs) and exclude them from generating BTF debugging information (when DEBUG_INFO_BTF is enabled). And if pahole doesn't support the --lang-exclude flag, we can't enable both RUST and DEBUG_INFO_BTF at the same time. So, in any case, the script is_rust_module.sh is just redundant and we can drop it. NOTE: we may also be able to drop the "Rust loadable module" mark inside Rust modules, but it seems safer to keep it for now to make sure we are not breaking any external tool that may potentially rely on it. Signed-off-by: Andrea Righi Reviewed-by: Nathan Chancellor Tested-by: Eric Curtin Reviewed-by: Eric Curtin Reviewed-by: Neal Gompa Reviewed-by: Masahiro Yamada Reviewed-by: Martin Rodriguez Reboredo Acked-by: Daniel Xu Link: https://lore.kernel.org/r/20230704052136.155445-1-andrea.righi@canonical.com [ Picked the `Reviewed-by`s from the old patch too. ] Signed-off-by: Miguel Ojeda --- scripts/Makefile.modfinal | 2 -- scripts/is_rust_module.sh | 16 ---------------- 2 files changed, 18 deletions(-) delete mode 100755 scripts/is_rust_module.sh (limited to 'scripts') diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal index fc19f67039bd..b3a6aa8fbe8c 100644 --- a/scripts/Makefile.modfinal +++ b/scripts/Makefile.modfinal @@ -41,8 +41,6 @@ quiet_cmd_btf_ko = BTF [M] $@ cmd_btf_ko = \ if [ ! -f vmlinux ]; then \ printf "Skipping BTF generation for %s due to unavailability of vmlinux\n" $@ 1>&2; \ - elif [ -n "$(CONFIG_RUST)" ] && $(srctree)/scripts/is_rust_module.sh $@; then \ - printf "Skipping BTF generation for %s because it's a Rust module\n" $@ 1>&2; \ else \ LLVM_OBJCOPY="$(OBJCOPY)" $(PAHOLE) -J $(PAHOLE_FLAGS) --btf_base vmlinux $@; \ $(RESOLVE_BTFIDS) -b vmlinux $@; \ diff --git a/scripts/is_rust_module.sh b/scripts/is_rust_module.sh deleted file mode 100755 index 464761a7cf7f..000000000000 --- a/scripts/is_rust_module.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 -# -# is_rust_module.sh module.ko -# -# Returns `0` if `module.ko` is a Rust module, `1` otherwise. - -set -e - -# Using the `16_` prefix ensures other symbols with the same substring -# are not picked up (even if it would be unlikely). The last part is -# used just in case LLVM decides to use the `.` suffix. -# -# In the future, checking for the `.comment` section may be another -# option, see https://github.com/rust-lang/rust/pull/97550. -${NM} "$*" | grep -qE '^[0-9a-fA-F]+ [Rr] _R[^[:space:]]+16___IS_RUST_MODULE[^[:space:]]*$' -- cgit From 89eed1ab1161e7d60595917e3b982e03dfcc0f8d Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sun, 30 Jul 2023 00:03:16 +0200 Subject: rust: upgrade to Rust 1.71.1 This is the second upgrade to the Rust toolchain, from 1.68.2 to 1.71.1 (i.e. the latest). See the upgrade policy [1] and the comments on the first upgrade in commit 3ed03f4da06e ("rust: upgrade to Rust 1.68.2"). # Unstable features No unstable features (that we use) were stabilized. Therefore, the only unstable feature allowed to be used outside the `kernel` crate is still `new_uninit`, though other code to be upstreamed may increase the list. Please see [2] for details. # Required changes For the upgrade, this patch requires the following changes: - Removal of the `__rust_*` allocator functions, together with the addition of the `__rust_no_alloc_shim_is_unstable` static. See [3] for details. - Some more compiler builtins added due to `::midpoint()` that got added in Rust 1.71 [4]. # `alloc` upgrade and reviewing The vast majority of changes are due to our `alloc` fork being upgraded at once. There are two kinds of changes to be aware of: the ones coming from upstream, which we should follow as closely as possible, and the updates needed in our added fallible APIs to keep them matching the newer infallible APIs coming from upstream. Instead of taking a look at the diff of this patch, an alternative approach is reviewing a diff of the changes between upstream `alloc` and the kernel's. This allows to easily inspect the kernel additions only, especially to check if the fallible methods we already have still match the infallible ones in the new version coming from upstream. Another approach is reviewing the changes introduced in the additions in the kernel fork between the two versions. This is useful to spot potentially unintended changes to our additions. To apply these approaches, one may follow steps similar to the following to generate a pair of patches that show the differences between upstream Rust and the kernel (for the subset of `alloc` we use) before and after applying this patch: # Get the difference with respect to the old version. git -C rust checkout $(linux/scripts/min-tool-version.sh rustc) git -C linux ls-tree -r --name-only HEAD -- rust/alloc | cut -d/ -f3- | grep -Fv README.md | xargs -IPATH cp rust/library/alloc/src/PATH linux/rust/alloc/PATH git -C linux diff --patch-with-stat --summary -R > old.patch git -C linux restore rust/alloc # Apply this patch. git -C linux am rust-upgrade.patch # Get the difference with respect to the new version. git -C rust checkout $(linux/scripts/min-tool-version.sh rustc) git -C linux ls-tree -r --name-only HEAD -- rust/alloc | cut -d/ -f3- | grep -Fv README.md | xargs -IPATH cp rust/library/alloc/src/PATH linux/rust/alloc/PATH git -C linux diff --patch-with-stat --summary -R > new.patch git -C linux restore rust/alloc Now one may check the `new.patch` to take a look at the additions (first approach) or at the difference between those two patches (second approach). For the latter, a side-by-side tool is recommended. Link: https://rust-for-linux.com/rust-version-policy [1] Link: https://github.com/Rust-for-Linux/linux/issues/2 [2] Link: https://github.com/rust-lang/rust/pull/86844 [3] Link: https://github.com/rust-lang/rust/pull/92048 [4] Closes: https://github.com/Rust-for-Linux/linux/issues/68 Reviewed-by: Martin Rodriguez Reboredo Reviewed-by: Trevor Gross Link: https://lore.kernel.org/r/20230729220317.416771-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/min-tool-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh index 2ade63149466..8da3b53fe36a 100755 --- a/scripts/min-tool-version.sh +++ b/scripts/min-tool-version.sh @@ -31,7 +31,7 @@ llvm) fi ;; rustc) - echo 1.68.2 + echo 1.71.1 ;; bindgen) echo 0.56.0 -- cgit From 08ab786556ff177086ce93b26daf2a58edd10968 Mon Sep 17 00:00:00 2001 From: Aakash Sen Sharma Date: Tue, 13 Jun 2023 01:13:11 +0530 Subject: rust: bindgen: upgrade to 0.65.1 In LLVM 16, anonymous items may return names like `(unnamed union at ..)` rather than empty names [1], which breaks Rust-enabled builds because bindgen assumed an empty name instead of detecting them via `clang_Cursor_isAnonymous` [2]: $ make rustdoc LLVM=1 CLIPPY=1 -j$(nproc) RUSTC L rust/core.o BINDGEN rust/bindings/bindings_generated.rs BINDGEN rust/bindings/bindings_helpers_generated.rs BINDGEN rust/uapi/uapi_generated.rs thread 'main' panicked at '"ftrace_branch_data_union_(anonymous_at__/_/include/linux/compiler_types_h_146_2)" is not a valid Ident', .../proc-macro2-1.0.24/src/fallback.rs:693:9 ... thread 'main' panicked at '"ftrace_branch_data_union_(anonymous_at__/_/include/linux/compiler_types_h_146_2)" is not a valid Ident', .../proc-macro2-1.0.24/src/fallback.rs:693:9 ... This was fixed in bindgen 0.62.0. Therefore, upgrade bindgen to a more recent version, 0.65.1, to support LLVM 16. Since bindgen 0.58.0 changed the `--{white,black}list-*` flags to `--{allow,block}list-*` [3], update them on our side too. In addition, bindgen 0.61.0 moved its CLI utility into a binary crate called `bindgen-cli` [4]. Thus update the installation command in the Quick Start guide. Moreover, bindgen 0.61.0 changed the default functionality to bind `size_t` to `usize` [5] and added the `--no-size_t-is-usize` flag to not bind `size_t` as `usize`. Then bindgen 0.65.0 removed the `--size_t-is-usize` flag [6]. Thus stop passing the flag to bindgen. Finally, bindgen 0.61.0 added support for the `noreturn` attribute (in its different forms) [7]. Thus remove the infinite loop in our Rust panic handler after calling `BUG()`, since bindgen now correctly generates a `BUG()` binding that returns `!` instead of `()`. Link: https://github.com/llvm/llvm-project/commit/19e984ef8f49bc3ccced15621989fa9703b2cd5b [1] Link: https://github.com/rust-lang/rust-bindgen/pull/2319 [2] Link: https://github.com/rust-lang/rust-bindgen/pull/1990 [3] Link: https://github.com/rust-lang/rust-bindgen/pull/2284 [4] Link: https://github.com/rust-lang/rust-bindgen/commit/cc78b6fdb6e829e5fb8fa1639f2182cb49333569 [5] Link: https://github.com/rust-lang/rust-bindgen/pull/2408 [6] Link: https://github.com/rust-lang/rust-bindgen/issues/2094 [7] Signed-off-by: Aakash Sen Sharma Closes: https://github.com/Rust-for-Linux/linux/issues/1013 Tested-by: Ariel Miculas Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/20230612194311.24826-1-aakashsensharma@gmail.com [ Reworded commit message. Mentioned the `bindgen-cli` binary crate change, linked to it and updated the Quick Start guide. Re-added a deleted "as" word in a code comment and reflowed comment to respect the maximum length. ] Signed-off-by: Miguel Ojeda --- scripts/min-tool-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh index 8da3b53fe36a..d65ab8bfeaf4 100755 --- a/scripts/min-tool-version.sh +++ b/scripts/min-tool-version.sh @@ -34,7 +34,7 @@ rustc) echo 1.71.1 ;; bindgen) - echo 0.56.0 + echo 0.65.1 ;; *) echo "$1: unknown tool" >&2 -- cgit From 4f353e0d1282dfe6b8082290fe8e606c5739a954 Mon Sep 17 00:00:00 2001 From: Martin Rodriguez Reboredo Date: Fri, 4 Aug 2023 14:14:39 -0300 Subject: scripts: generate_rust_analyzer: provide `cfg`s for `core` and `alloc` Both `core` and `alloc` have their `cfgs` (such as `no_rc`) missing in `rust-project.json`. To remedy this, pass the flags to `generate_rust_analyzer.py` for them to be added to a dictionary where each key corresponds to a crate and each value to a list of `cfg`s. The dictionary is then used to pass the `cfg`s to each crate in the generated file (for `core` and `alloc` only). Signed-off-by: Martin Rodriguez Reboredo Link: https://lore.kernel.org/r/20230804171448.54976-1-yakoyoku@gmail.com [ Removed `Suggested-by` as discussed in mailing list. ] Signed-off-by: Miguel Ojeda --- scripts/generate_rust_analyzer.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index 848fa1ad92ba..fc52bc41d3e7 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -10,7 +10,15 @@ import os import pathlib import sys -def generate_crates(srctree, objtree, sysroot_src, external_src): +def args_crates_cfgs(cfgs): + crates_cfgs = {} + for cfg in cfgs: + crate, vals = cfg.split("=", 1) + crates_cfgs[crate] = vals.replace("--cfg", "").split() + + return crates_cfgs + +def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs): # Generate the configuration list. cfg = [] with open(objtree / "include" / "generated" / "rustc_cfg") as fd: @@ -24,6 +32,7 @@ def generate_crates(srctree, objtree, sysroot_src, external_src): # Avoid O(n^2) iterations by keeping a map of indexes. crates = [] crates_indexes = {} + crates_cfgs = args_crates_cfgs(cfgs) def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=True, is_proc_macro=False): crates_indexes[display_name] = len(crates) @@ -45,6 +54,7 @@ def generate_crates(srctree, objtree, sysroot_src, external_src): "core", sysroot_src / "core" / "src" / "lib.rs", [], + cfg=crates_cfgs.get("core", []), is_workspace_member=False, ) @@ -58,6 +68,7 @@ def generate_crates(srctree, objtree, sysroot_src, external_src): "alloc", srctree / "rust" / "alloc" / "lib.rs", ["core", "compiler_builtins"], + cfg=crates_cfgs.get("alloc", []), ) append_crate( @@ -131,6 +142,7 @@ def generate_crates(srctree, objtree, sysroot_src, external_src): def main(): parser = argparse.ArgumentParser() parser.add_argument('--verbose', '-v', action='store_true') + parser.add_argument('--cfgs', action='append', default=[]) parser.add_argument("srctree", type=pathlib.Path) parser.add_argument("objtree", type=pathlib.Path) parser.add_argument("sysroot_src", type=pathlib.Path) @@ -143,7 +155,7 @@ def main(): ) rust_project = { - "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree), + "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs), "sysroot_src": str(args.sysroot_src), } -- cgit