From 392885ee82d35d515ba2af7b72c5e357c3002113 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 30 Nov 2018 10:05:22 +0900 Subject: kbuild: let fixdep directly write to .*.cmd files Currently, fixdep writes dependencies to .*.tmp, which is renamed to .*.cmd after everything succeeds. This is a very safe way to avoid corrupted .*.cmd files. The if_changed_dep has carried this safety mechanism since it was added in 2002. If fixdep fails for some reasons or a user terminates the build while fixdep is running, the incomplete output from the fixdep could be troublesome. This is my insight about some bad scenarios: [1] If the compiler succeeds to generate *.o file, but fixdep fails to write necessary dependencies to .*.cmd file, Make will miss to rebuild the object when headers or CONFIG options are changed. In this case, fixdep should not generate .*.cmd file at all so that 'arg-check' will surely trigger the rebuild of the object. [2] A partially constructed .*.cmd file may not be a syntactically correct makefile. The next time Make runs, it would include it, then fail to parse it. Once this happens, 'make clean' is be the only way to fix it. In fact, [1] is no longer a problem since commit 9c2af1c7377a ("kbuild: add .DELETE_ON_ERROR special target"). Make deletes a target file on any failure in its recipe. Because fixdep is a part of the recipe of *.o target, if it fails, the *.o is deleted anyway. However, I am a bit worried about the slight possibility of [2]. So, here is a solution. Let fixdep directly write to a .*.cmd file, but allow makefiles to include it only when its corresponding target exists. This effectively reverts commit 2982c953570b ("kbuild: remove redundant $(wildcard ...) for cmd_files calculation"), and commit 00d78ab2ba75 ("kbuild: remove dead code in cmd_files calculation in top Makefile") because now we must check the presence of targets. Signed-off-by: Masahiro Yamada --- scripts/Kbuild.include | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'scripts/Kbuild.include') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index bb015551c2d9..6cf6a8b83b97 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -264,9 +264,8 @@ ifndef CONFIG_TRIM_UNUSED_KSYMS cmd_and_fixdep = \ $(echo-cmd) $(cmd_$(1)); \ - scripts/basic/fixdep $(depfile) $@ '$(make-cmd)' > $(dot-target).tmp;\ - rm -f $(depfile); \ - mv -f $(dot-target).tmp $(dot-target).cmd; + scripts/basic/fixdep $(depfile) $@ '$(make-cmd)' > $(dot-target).cmd;\ + rm -f $(depfile); else @@ -289,9 +288,8 @@ cmd_and_fixdep = \ $(echo-cmd) $(cmd_$(1)); \ $(ksym_dep_filter) | \ scripts/basic/fixdep -e $(depfile) $@ '$(make-cmd)' \ - > $(dot-target).tmp; \ - rm -f $(depfile); \ - mv -f $(dot-target).tmp $(dot-target).cmd; + > $(dot-target).cmd; \ + rm -f $(depfile); endif -- cgit From bbda5ec671d3fe62faefa1cab7270aa586042a4b Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 30 Nov 2018 10:05:26 +0900 Subject: kbuild: simplify dependency generation for CONFIG_TRIM_UNUSED_KSYMS My main motivation of this commit is to clean up scripts/Kbuild.include and scripts/Makefile.build. Currently, CONFIG_TRIM_UNUSED_KSYMS works with a tricky gimmick; possibly exported symbols are detected by letting $(CPP) replace EXPORT_SYMBOL* with a special string '=== __KSYM_*===', which is post-processed by sed, and passed to fixdep. The extra preprocessing is costly, and hacking cmd_and_fixdep is ugly. I came up with a new way to find exported symbols; insert a dummy symbol __ksym_marker_* to each potentially exported symbol. Those dummy symbols are picked up by $(NM), post-processed by sed, then appended to .*.cmd files. I collected the post-process part to a new shell script scripts/gen_ksymdeps.sh for readability. The dummy symbols are put into the .discard.* section so that the linker script rips them off the final vmlinux or modules. A nice side-effect is building with CONFIG_TRIM_UNUSED_KSYMS will be much faster. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Pitre --- scripts/Kbuild.include | 28 ---------------------------- 1 file changed, 28 deletions(-) (limited to 'scripts/Kbuild.include') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 6cf6a8b83b97..4b943f4d2226 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -260,39 +260,11 @@ if_changed_dep = $(if $(strip $(any-prereq) $(arg-check) ), \ @set -e; \ $(cmd_and_fixdep), @:) -ifndef CONFIG_TRIM_UNUSED_KSYMS - cmd_and_fixdep = \ $(echo-cmd) $(cmd_$(1)); \ scripts/basic/fixdep $(depfile) $@ '$(make-cmd)' > $(dot-target).cmd;\ rm -f $(depfile); -else - -# Filter out exported kernel symbol names from the preprocessor output. -# See also __KSYM_DEPS__ in include/linux/export.h. -# We disable the depfile generation here, so as not to overwrite the existing -# depfile while fixdep is parsing it. -flags_nodeps = $(filter-out -Wp$(comma)-M%, $($(1))) -ksym_dep_filter = \ - case "$(1)" in \ - cc_*_c|cpp_i_c) \ - $(CPP) $(call flags_nodeps,c_flags) -D__KSYM_DEPS__ $< ;; \ - as_*_S|cpp_s_S) \ - $(CPP) $(call flags_nodeps,a_flags) -D__KSYM_DEPS__ $< ;; \ - boot*|build*|cpp_its_S|*cpp_lds_S|dtc|host*|vdso*) : ;; \ - *) echo "Don't know how to preprocess $(1)" >&2; false ;; \ - esac | tr ";" "\n" | sed -n 's/^.*=== __KSYM_\(.*\) ===.*$$/_\1/p' - -cmd_and_fixdep = \ - $(echo-cmd) $(cmd_$(1)); \ - $(ksym_dep_filter) | \ - scripts/basic/fixdep -e $(depfile) $@ '$(make-cmd)' \ - > $(dot-target).cmd; \ - rm -f $(depfile); - -endif - # Usage: $(call if_changed_rule,foo) # Will check if $(cmd_foo) or any of the prerequisites changed, # and if so will execute $(rule_foo). -- cgit From 3a2429e1faf40b2aaa481aa4b001a74d222c7e8b Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 30 Nov 2018 10:05:27 +0900 Subject: kbuild: change if_changed_rule for multi-line recipe The 'define' ... 'endef' directive is useful to confine a series of shell commands into a single macro: define foo [action1] [action2] [action3] endif Each action is executed in a separate subshell. However, rule_cc_o_c and rule_as_o_S in scripts/Makefile.build are written as follows (with a trailing semicolon in each cmd_*): define rule_cc_o_c [action1] ; \ [action2] ; \ [action3] ; endef All shell commands are concatenated with '; \' so that it looks like a single command from the Makefile point of view. This does not exploit the benefits of 'define' ... 'endef' form because a single shell command can be more simply written, like this: rule_cc_o_c = \ [action1] ; \ [action2] ; \ [action3] ; I guess the intention for the command concatenation was to let the '@set -e' in if_changed_rule cover all the commands. We can improve the readability by moving '@set -e' to the 'cmd' macro. The combo of $(call echo-cmd,*) $(cmd_*) in rule_cc_o_c and rule_as_o_S have been replaced with $(call cmd,*). The trailing back-slashes have been removed. Here is a note about the performance: the commands in rule_cc_o_c and rule_as_o_S were previously executed all together in a single subshell, but now each line in a separate subshell. This means Make will spawn extra subshells [1]. I measured the build performance for x86_64_defconfig + CONFIG_MODVERSIONS + CONFIG_TRIM_UNUSED_KSYMS and I saw slight performance regression, but I believe code readability and maintainability wins. [1] Precisely, GNU Make may optimize this by executing the command directly instead of forking a subshell, if no shell special characters are found in the command line and omitting the subshell will not change the behavior. Signed-off-by: Masahiro Yamada --- scripts/Kbuild.include | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'scripts/Kbuild.include') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 4b943f4d2226..6bbbe7533d6f 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -215,7 +215,7 @@ echo-cmd = $(if $($(quiet)cmd_$(1)),\ echo ' $(call escsq,$($(quiet)cmd_$(1)))$(echo-why)';) # printing commands -cmd = @$(echo-cmd) $(cmd_$(1)) +cmd = @set -e; $(echo-cmd) $(cmd_$(1)) # Add $(obj)/ for paths that are not absolute objectify = $(foreach o,$(1),$(if $(filter /%,$(o)),$(o),$(obj)/$(o))) @@ -256,21 +256,17 @@ if_changed = $(if $(strip $(any-prereq) $(arg-check)), \ printf '%s\n' 'cmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:) # Execute the command and also postprocess generated .d dependencies file. -if_changed_dep = $(if $(strip $(any-prereq) $(arg-check) ), \ - @set -e; \ - $(cmd_and_fixdep), @:) +if_changed_dep = $(if $(strip $(any-prereq) $(arg-check)),$(cmd_and_fixdep),@:) cmd_and_fixdep = \ - $(echo-cmd) $(cmd_$(1)); \ + $(cmd); \ scripts/basic/fixdep $(depfile) $@ '$(make-cmd)' > $(dot-target).cmd;\ rm -f $(depfile); # Usage: $(call if_changed_rule,foo) # Will check if $(cmd_foo) or any of the prerequisites changed, # and if so will execute $(rule_foo). -if_changed_rule = $(if $(strip $(any-prereq) $(arg-check) ), \ - @set -e; \ - $(rule_$(1)), @:) +if_changed_rule = $(if $(strip $(any-prereq) $(arg-check)),$(rule_$(1)),@:) ### # why - tell why a target got built -- cgit From e5d289100d3aef3e83dc1b8303999e26d86acb7b Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 30 Nov 2018 10:05:28 +0900 Subject: kbuild: remove trailing semicolon from cmd_* passed to if_changed_rule With the change of rule_cc_o_c / rule_as_o_S in the last commit, each command is executed in a separate subshell. Rip off unneeded semicolons. Signed-off-by: Masahiro Yamada --- scripts/Kbuild.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/Kbuild.include') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 6bbbe7533d6f..ba297fdd7c52 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -261,7 +261,7 @@ if_changed_dep = $(if $(strip $(any-prereq) $(arg-check)),$(cmd_and_fixdep),@:) cmd_and_fixdep = \ $(cmd); \ scripts/basic/fixdep $(depfile) $@ '$(make-cmd)' > $(dot-target).cmd;\ - rm -f $(depfile); + rm -f $(depfile) # Usage: $(call if_changed_rule,foo) # Will check if $(cmd_foo) or any of the prerequisites changed, -- cgit From 67126965ee0856f0d1cdb10792a36eaf4b2f13c0 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 30 Nov 2018 10:05:29 +0900 Subject: kbuild: refactor if_changed '@set -e; $(echo-cmd) $(cmd_$(1)' can be replaced with '$(cmd)'. Signed-off-by: Masahiro Yamada --- scripts/Kbuild.include | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'scripts/Kbuild.include') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index ba297fdd7c52..87e188609ef7 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -251,8 +251,7 @@ any-prereq = $(filter-out $(PHONY),$?) $(filter-out $(PHONY) $(wildcard $^),$^) # Execute command if command has changed or prerequisite(s) are updated. if_changed = $(if $(strip $(any-prereq) $(arg-check)), \ - @set -e; \ - $(echo-cmd) $(cmd_$(1)); \ + $(cmd); \ printf '%s\n' 'cmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:) # Execute the command and also postprocess generated .d dependencies file. -- cgit From 8e9b61b293d98f878cf1e6d1ae164e41c0219959 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 1 Dec 2018 09:27:15 +0900 Subject: kbuild: move .SECONDARY special target to Kbuild.include In commit 54a702f70589 ("kbuild: mark $(targets) as .SECONDARY and remove .PRECIOUS markers"), I missed one important feature of the .SECONDARY target: .SECONDARY with no prerequisites causes all targets to be treated as secondary. ... which agrees with the policy of Kbuild. Let's move it to scripts/Kbuild.include, with no prerequisites. Note: If an intermediate file is generated by $(call if_changed,...), you still need to add it to "targets" so its .*.cmd file is included. The arm/arm64 crypto files are generated by $(call cmd,shipped), so they do not need to be added to "targets", but need to be added to "clean-files" so "make clean" can properly clean them away. Signed-off-by: Masahiro Yamada --- scripts/Kbuild.include | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts/Kbuild.include') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 87e188609ef7..4f138620865b 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -358,3 +358,6 @@ endef # delete partially updated (i.e. corrupted) files on error .DELETE_ON_ERROR: + +# do not delete intermediate files automatically +.SECONDARY: -- cgit