From f7b101d33046a837c2aa4526cef28a3c785d7af2 Mon Sep 17 00:00:00 2001 From: "Joel Fernandes (Google)" Date: Wed, 15 May 2019 17:35:51 -0400 Subject: kheaders: Move from proc to sysfs The kheaders archive consisting of the kernel headers used for compiling bpf programs is in /proc. However there is concern that moving it here will make it permanent. Let us move it to /sys/kernel as discussed [1]. [1] https://lore.kernel.org/patchwork/patch/1067310/#1265969 Suggested-by: Steven Rostedt Signed-off-by: Joel Fernandes (Google) Signed-off-by: Greg Kroah-Hartman --- kernel/Makefile | 4 +-- kernel/gen_ikh_data.sh | 89 -------------------------------------------------- kernel/gen_kheaders.sh | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ kernel/kheaders.c | 40 +++++++++-------------- 4 files changed, 107 insertions(+), 115 deletions(-) delete mode 100755 kernel/gen_ikh_data.sh create mode 100755 kernel/gen_kheaders.sh (limited to 'kernel') diff --git a/kernel/Makefile b/kernel/Makefile index 33824f0385b3..a8d923b5481b 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -71,7 +71,7 @@ obj-$(CONFIG_UTS_NS) += utsname.o obj-$(CONFIG_USER_NS) += user_namespace.o obj-$(CONFIG_PID_NS) += pid_namespace.o obj-$(CONFIG_IKCONFIG) += configs.o -obj-$(CONFIG_IKHEADERS_PROC) += kheaders.o +obj-$(CONFIG_IKHEADERS) += kheaders.o obj-$(CONFIG_SMP) += stop_machine.o obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o obj-$(CONFIG_AUDIT) += audit.o auditfilter.o @@ -127,7 +127,7 @@ $(obj)/config_data.gz: $(KCONFIG_CONFIG) FORCE $(obj)/kheaders.o: $(obj)/kheaders_data.tar.xz quiet_cmd_genikh = CHK $(obj)/kheaders_data.tar.xz -cmd_genikh = $(CONFIG_SHELL) $(srctree)/kernel/gen_ikh_data.sh $@ +cmd_genikh = $(CONFIG_SHELL) $(srctree)/kernel/gen_kheaders.sh $@ $(obj)/kheaders_data.tar.xz: FORCE $(call cmd,genikh) diff --git a/kernel/gen_ikh_data.sh b/kernel/gen_ikh_data.sh deleted file mode 100755 index 591a94f7b387..000000000000 --- a/kernel/gen_ikh_data.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0 - -# This script generates an archive consisting of kernel headers -# for CONFIG_IKHEADERS_PROC. -set -e -spath="$(dirname "$(readlink -f "$0")")" -kroot="$spath/.." -outdir="$(pwd)" -tarfile=$1 -cpio_dir=$outdir/$tarfile.tmp - -# Script filename relative to the kernel source root -# We add it to the archive because it is small and any changes -# to this script will also cause a rebuild of the archive. -sfile="$(realpath --relative-to $kroot "$(readlink -f "$0")")" - -src_file_list=" -include/ -arch/$SRCARCH/include/ -$sfile -" - -obj_file_list=" -include/ -arch/$SRCARCH/include/ -" - -# Support incremental builds by skipping archive generation -# if timestamps of files being archived are not changed. - -# This block is useful for debugging the incremental builds. -# Uncomment it for debugging. -# iter=1 -# if [ ! -f /tmp/iter ]; then echo 1 > /tmp/iter; -# else; iter=$(($(cat /tmp/iter) + 1)); fi -# find $src_file_list -type f | xargs ls -lR > /tmp/src-ls-$iter -# find $obj_file_list -type f | xargs ls -lR > /tmp/obj-ls-$iter - -# include/generated/compile.h is ignored because it is touched even when none -# of the source files changed. This causes pointless regeneration, so let us -# ignore them for md5 calculation. -pushd $kroot > /dev/null -src_files_md5="$(find $src_file_list -type f | - grep -v "include/generated/compile.h" | - xargs ls -lR | md5sum | cut -d ' ' -f1)" -popd > /dev/null -obj_files_md5="$(find $obj_file_list -type f | - grep -v "include/generated/compile.h" | - xargs ls -lR | md5sum | cut -d ' ' -f1)" - -if [ -f $tarfile ]; then tarfile_md5="$(md5sum $tarfile | cut -d ' ' -f1)"; fi -if [ -f kernel/kheaders.md5 ] && - [ "$(cat kernel/kheaders.md5|head -1)" == "$src_files_md5" ] && - [ "$(cat kernel/kheaders.md5|head -2|tail -1)" == "$obj_files_md5" ] && - [ "$(cat kernel/kheaders.md5|tail -1)" == "$tarfile_md5" ]; then - exit -fi - -if [ "${quiet}" != "silent_" ]; then - echo " GEN $tarfile" -fi - -rm -rf $cpio_dir -mkdir $cpio_dir - -pushd $kroot > /dev/null -for f in $src_file_list; - do find "$f" ! -name "*.cmd" ! -name ".*"; -done | cpio --quiet -pd $cpio_dir -popd > /dev/null - -# The second CPIO can complain if files already exist which can -# happen with out of tree builds. Just silence CPIO for now. -for f in $obj_file_list; - do find "$f" ! -name "*.cmd" ! -name ".*"; -done | cpio --quiet -pd $cpio_dir >/dev/null 2>&1 - -# Remove comments except SDPX lines -find $cpio_dir -type f -print0 | - xargs -0 -P8 -n1 perl -pi -e 'BEGIN {undef $/;}; s/\/\*((?!SPDX).)*?\*\///smg;' - -tar -Jcf $tarfile -C $cpio_dir/ . > /dev/null - -echo "$src_files_md5" > kernel/kheaders.md5 -echo "$obj_files_md5" >> kernel/kheaders.md5 -echo "$(md5sum $tarfile | cut -d ' ' -f1)" >> kernel/kheaders.md5 - -rm -rf $cpio_dir diff --git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh new file mode 100755 index 000000000000..581b83534587 --- /dev/null +++ b/kernel/gen_kheaders.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# This script generates an archive consisting of kernel headers +# for CONFIG_IKHEADERS. +set -e +spath="$(dirname "$(readlink -f "$0")")" +kroot="$spath/.." +outdir="$(pwd)" +tarfile=$1 +cpio_dir=$outdir/$tarfile.tmp + +# Script filename relative to the kernel source root +# We add it to the archive because it is small and any changes +# to this script will also cause a rebuild of the archive. +sfile="$(realpath --relative-to $kroot "$(readlink -f "$0")")" + +src_file_list=" +include/ +arch/$SRCARCH/include/ +$sfile +" + +obj_file_list=" +include/ +arch/$SRCARCH/include/ +" + +# Support incremental builds by skipping archive generation +# if timestamps of files being archived are not changed. + +# This block is useful for debugging the incremental builds. +# Uncomment it for debugging. +# iter=1 +# if [ ! -f /tmp/iter ]; then echo 1 > /tmp/iter; +# else; iter=$(($(cat /tmp/iter) + 1)); fi +# find $src_file_list -type f | xargs ls -lR > /tmp/src-ls-$iter +# find $obj_file_list -type f | xargs ls -lR > /tmp/obj-ls-$iter + +# include/generated/compile.h is ignored because it is touched even when none +# of the source files changed. This causes pointless regeneration, so let us +# ignore them for md5 calculation. +pushd $kroot > /dev/null +src_files_md5="$(find $src_file_list -type f | + grep -v "include/generated/compile.h" | + xargs ls -lR | md5sum | cut -d ' ' -f1)" +popd > /dev/null +obj_files_md5="$(find $obj_file_list -type f | + grep -v "include/generated/compile.h" | + xargs ls -lR | md5sum | cut -d ' ' -f1)" + +if [ -f $tarfile ]; then tarfile_md5="$(md5sum $tarfile | cut -d ' ' -f1)"; fi +if [ -f kernel/kheaders.md5 ] && + [ "$(cat kernel/kheaders.md5|head -1)" == "$src_files_md5" ] && + [ "$(cat kernel/kheaders.md5|head -2|tail -1)" == "$obj_files_md5" ] && + [ "$(cat kernel/kheaders.md5|tail -1)" == "$tarfile_md5" ]; then + exit +fi + +if [ "${quiet}" != "silent_" ]; then + echo " GEN $tarfile" +fi + +rm -rf $cpio_dir +mkdir $cpio_dir + +pushd $kroot > /dev/null +for f in $src_file_list; + do find "$f" ! -name "*.cmd" ! -name ".*"; +done | cpio --quiet -pd $cpio_dir +popd > /dev/null + +# The second CPIO can complain if files already exist which can +# happen with out of tree builds. Just silence CPIO for now. +for f in $obj_file_list; + do find "$f" ! -name "*.cmd" ! -name ".*"; +done | cpio --quiet -pd $cpio_dir >/dev/null 2>&1 + +# Remove comments except SDPX lines +find $cpio_dir -type f -print0 | + xargs -0 -P8 -n1 perl -pi -e 'BEGIN {undef $/;}; s/\/\*((?!SPDX).)*?\*\///smg;' + +tar -Jcf $tarfile -C $cpio_dir/ . > /dev/null + +echo "$src_files_md5" > kernel/kheaders.md5 +echo "$obj_files_md5" >> kernel/kheaders.md5 +echo "$(md5sum $tarfile | cut -d ' ' -f1)" >> kernel/kheaders.md5 + +rm -rf $cpio_dir diff --git a/kernel/kheaders.c b/kernel/kheaders.c index 70ae6052920d..8f69772af77b 100644 --- a/kernel/kheaders.c +++ b/kernel/kheaders.c @@ -8,9 +8,8 @@ #include #include -#include +#include #include -#include /* * Define kernel_headers_data and kernel_headers_data_end, within which the @@ -31,39 +30,32 @@ extern char kernel_headers_data; extern char kernel_headers_data_end; static ssize_t -ikheaders_read_current(struct file *file, char __user *buf, - size_t len, loff_t *offset) +ikheaders_read(struct file *file, struct kobject *kobj, + struct bin_attribute *bin_attr, + char *buf, loff_t off, size_t len) { - return simple_read_from_buffer(buf, len, offset, - &kernel_headers_data, - &kernel_headers_data_end - - &kernel_headers_data); + memcpy(buf, &kernel_headers_data + off, len); + return len; } -static const struct file_operations ikheaders_file_ops = { - .read = ikheaders_read_current, - .llseek = default_llseek, +static struct bin_attribute kheaders_attr __ro_after_init = { + .attr = { + .name = "kheaders.tar.xz", + .mode = 0444, + }, + .read = &ikheaders_read, }; static int __init ikheaders_init(void) { - struct proc_dir_entry *entry; - - /* create the current headers file */ - entry = proc_create("kheaders.tar.xz", S_IRUGO, NULL, - &ikheaders_file_ops); - if (!entry) - return -ENOMEM; - - proc_set_size(entry, - &kernel_headers_data_end - - &kernel_headers_data); - return 0; + kheaders_attr.size = (&kernel_headers_data_end - + &kernel_headers_data); + return sysfs_create_bin_file(kernel_kobj, &kheaders_attr); } static void __exit ikheaders_cleanup(void) { - remove_proc_entry("kheaders.tar.xz", NULL); + sysfs_remove_bin_file(kernel_kobj, &kheaders_attr); } module_init(ikheaders_init); -- cgit From 1457dc9ed8da871fbbc0a2ebdaed0405eeeed0cf Mon Sep 17 00:00:00 2001 From: "Joel Fernandes (Google)" Date: Wed, 15 May 2019 17:35:52 -0400 Subject: kheaders: Do not regenerate archive if config is not changed Linus reported an issue that doing an allmodconfig was causing the kheaders archive to be regenerated even though the config is the same. This patch fixes the issue by ignoring the config-related header files for "knowing when to regenerate based on timestamps". Instead, if the CONFIG_X_Y option really changes, then we there are the include/config/X/Y.h which will already tells us "if a config really changed". So we don't really need these files for regeneration detection anyway, and ignoring them fixes Linus's issue. Reported-by: Linus Torvalds Signed-off-by: Joel Fernandes (Google) Signed-off-by: Greg Kroah-Hartman --- kernel/gen_kheaders.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh index 581b83534587..9a34e1d9bd7f 100755 --- a/kernel/gen_kheaders.sh +++ b/kernel/gen_kheaders.sh @@ -31,9 +31,8 @@ arch/$SRCARCH/include/ # This block is useful for debugging the incremental builds. # Uncomment it for debugging. -# iter=1 -# if [ ! -f /tmp/iter ]; then echo 1 > /tmp/iter; -# else; iter=$(($(cat /tmp/iter) + 1)); fi +# if [ ! -f /tmp/iter ]; then iter=1; echo 1 > /tmp/iter; +# else iter=$(($(cat /tmp/iter) + 1)); echo $iter > /tmp/iter; fi # find $src_file_list -type f | xargs ls -lR > /tmp/src-ls-$iter # find $obj_file_list -type f | xargs ls -lR > /tmp/obj-ls-$iter @@ -43,10 +42,18 @@ arch/$SRCARCH/include/ pushd $kroot > /dev/null src_files_md5="$(find $src_file_list -type f | grep -v "include/generated/compile.h" | + grep -v "include/generated/autoconf.h" | + grep -v "include/config/auto.conf" | + grep -v "include/config/auto.conf.cmd" | + grep -v "include/config/tristate.conf" | xargs ls -lR | md5sum | cut -d ' ' -f1)" popd > /dev/null obj_files_md5="$(find $obj_file_list -type f | grep -v "include/generated/compile.h" | + grep -v "include/generated/autoconf.h" | + grep -v "include/config/auto.conf" | + grep -v "include/config/auto.conf.cmd" | + grep -v "include/config/tristate.conf" | xargs ls -lR | md5sum | cut -d ' ' -f1)" if [ -f $tarfile ]; then tarfile_md5="$(md5sum $tarfile | cut -d ' ' -f1)"; fi @@ -82,7 +89,7 @@ find $cpio_dir -type f -print0 | tar -Jcf $tarfile -C $cpio_dir/ . > /dev/null -echo "$src_files_md5" > kernel/kheaders.md5 +echo "$src_files_md5" > kernel/kheaders.md5 echo "$obj_files_md5" >> kernel/kheaders.md5 echo "$(md5sum $tarfile | cut -d ' ' -f1)" >> kernel/kheaders.md5 -- cgit