#!/bin/bash
# SPDX-License-Identifier: GPL-2.0+
#
# Run a series of torture tests, intended for overnight or
# longer timeframes, and also for large systems.
#
# Usage: torture.sh [ options ]
#
# Copyright (C) 2020 Facebook, Inc.
#
# Authors: Paul E. McKenney <paulmck@kernel.org>

scriptname=$0
args="$*"

KVM="`pwd`/tools/testing/selftests/rcutorture"; export KVM
PATH=${KVM}/bin:$PATH; export PATH
. functions.sh

TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`"
MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2))
HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2))
if test "$HALF_ALLOTED_CPUS" -lt 1
then
	HALF_ALLOTED_CPUS=1
fi
VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16))
if test "$VERBOSE_BATCH_CPUS" -lt 2
then
	VERBOSE_BATCH_CPUS=0
fi

# Default duration and apportionment.
duration_base=10
duration_rcutorture_frac=7
duration_locktorture_frac=1
duration_scftorture_frac=2

# "yes" or "no" parameters
do_allmodconfig=yes
do_rcutorture=yes
do_locktorture=yes
do_scftorture=yes
do_rcuscale=yes
do_refscale=yes
do_kvfree=yes
do_kasan=yes
do_kcsan=no

# doyesno - Helper function for yes/no arguments
function doyesno () {
	if test "$1" = "$2"
	then
		echo yes
	else
		echo no
	fi
}

usage () {
	echo "Usage: $scriptname optional arguments:"
	echo "       --doall"
	echo "       --doallmodconfig / --do-no-allmodconfig"
	echo "       --do-kasan / --do-no-kasan"
	echo "       --do-kcsan / --do-no-kcsan"
	echo "       --do-kvfree / --do-no-kvfree"
	echo "       --do-locktorture / --do-no-locktorture"
	echo "       --do-none"
	echo "       --do-rcuscale / --do-no-rcuscale"
	echo "       --do-rcutorture / --do-no-rcutorture"
	echo "       --do-refscale / --do-no-refscale"
	echo "       --do-scftorture / --do-no-scftorture"
	echo "       --duration [ <minutes> | <hours>h | <days>d ]"
	exit 1
}

while test $# -gt 0
do
	case "$1" in
	--doall)
		do_allmodconfig=yes
		do_rcutorture=yes
		do_locktorture=yes
		do_scftorture=yes
		do_rcuscale=yes
		do_refscale=yes
		do_kvfree=yes
		do_kasan=yes
		do_kcsan=yes
		;;
	--do-allmodconfig|--do-no-allmodconfig)
		do_allmodconfig=`doyesno "$1" --do-allmodconfig`
		;;
	--do-kasan|--do-no-kasan)
		do_kasan=`doyesno "$1" --do-kasan`
		;;
	--do-kcsan|--do-no-kcsan)
		do_kcsan=`doyesno "$1" --do-kcsan`
		;;
	--do-kvfree|--do-no-kvfree)
		do_kvfree=`doyesno "$1" --do-kvfree`
		;;
	--do-locktorture|--do-no-locktorture)
		do_locktorture=`doyesno "$1" --do-locktorture`
		;;
	--do-none)
		do_allmodconfig=no
		do_rcutorture=no
		do_locktorture=no
		do_scftorture=no
		do_rcuscale=no
		do_refscale=no
		do_kvfree=no
		do_kasan=no
		do_kcsan=no
		;;
	--do-rcuscale|--do-no-rcuscale)
		do_rcuscale=`doyesno "$1" --do-rcuscale`
		;;
	--do-rcutorture|--do-no-rcutorture)
		do_rcutorture=`doyesno "$1" --do-rcutorture`
		;;
	--do-refscale|--do-no-refscale)
		do_refscale=`doyesno "$1" --do-refscale`
		;;
	--do-scftorture|--do-no-scftorture)
		do_scftorture=`doyesno "$1" --do-scftorture`
		;;
	--duration)
		checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error'
		mult=1
		if echo "$2" | grep -q 'm$'
		then
			mult=1
		elif echo "$2" | grep -q 'h$'
		then
			mult=60
		elif echo "$2" | grep -q 'd$'
		then
			mult=1440
		fi
		ts=`echo $2 | sed -e 's/[smhd]$//'`
		duration_base=$(($ts*mult))
		shift
		;;
	*)
		echo Unknown argument $1
		usage
		;;
	esac
	shift
done

duration_rcutorture=$((duration_base*duration_rcutorture_frac/10))
# Need to sum remaining weights, and if duration weights to zero,
# set do_no_rcutorture. @@@
duration_locktorture=$((duration_base*duration_locktorture_frac/10))
duration_scftorture=$((duration_base*duration_scftorture_frac/10))

T=/tmp/torture.sh.$$
trap 'rm -rf $T' 0 2
mkdir $T

touch $T/failures
touch $T/successes

ds="`date +%Y.%m.%d-%H.%M.%S`-torture"
startdate="`date`"
starttime="`get_starttime`"

# torture_one - Does a single kvm.sh run.
#
# Usage:
#	torture_bootargs="[ kernel boot arguments ]"
#	torture_one flavor [ kvm.sh arguments ]
#
# Note that "flavor" is an arbitrary string.  Supply --torture if needed.
# Note that quoting is problematic.  So on the command line, pass multiple
# values with multiple kvm.sh argument instances.
function torture_one {
	local cur_bootargs=
	local boottag=

	echo " --- $curflavor:" Start `date` | tee -a $T/log
	if test -n "$torture_bootargs"
	then
		boottag="--bootargs"
		cur_bootargs="$torture_bootargs"
	fi
	"$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1
	retcode=$?
	resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`"
	if test -n "$resdir"
	then
		cp $T/$curflavor.out $resdir/log.long
		echo retcode=$retcode >> $resdir/log.long
	else
		cat $T/$curflavor.out | tee -a $T/log
		echo retcode=$retcode | tee -a $T/log
	fi
	if test "$retcode" == 0
	then
		echo "$curflavor($retcode)" $resdir >> $T/successes
	else
		echo "$curflavor($retcode)" $resdir >> $T/failures
	fi
}

# torture_set - Does a set of tortures with and without KASAN and KCSAN.
#
# Usage:
#	torture_bootargs="[ kernel boot arguments ]"
#	torture_set flavor [ kvm.sh arguments ]
#
# Note that "flavor" is an arbitrary string.  Supply --torture if needed.
# Note that quoting is problematic.  So on the command line, pass multiple
# values with multiple kvm.sh argument instances.
function torture_set {
	local flavor=$1
	shift
	curflavor=$flavor
	torture_one "$@"
	if test "$do_kasan" = "yes"
	then
		curflavor=${flavor}-kasan
		torture_one "$@" --kasan
	fi
	if test "$do_kcsan" = "yes"
	then
		curflavor=${flavor}-kcsan
		torture_one $* --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" --kmake-arg "CC=clang" --kcsan
	fi
}

# make allmodconfig
if test "$do_allmodconfig" = "yes"
then
	echo " --- allmodconfig:" Start `date` | tee -a $T/log
	amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig"
	mkdir -p "$amcdir"
	make -j$MAKE_ALLOTED_CPUS clean > "$amcdir/Make.out" 2>&1
	make -j$MAKE_ALLOTED_CPUS allmodconfig > "$amcdir/Make.out" 2>&1
	make -j$MAKE_ALLOTED_CPUS > "$amcdir/Make.out" 2>&1
	retcode="$?"
	echo $retcode > "$amcdir/Make.exitcode"
	if test "$retcode" == 0
	then
		echo "allmodconfig($retcode)" $amcdir >> $T/successes
	else
		echo "allmodconfig($retcode)" $amcdir >> $T/failures
	fi
fi

# --torture rcu
if test "$do_rcutorture" = "yes"
then
	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
	torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE10 4*CFLIST" --trust-make
fi

if test "$do_locktorture" = "yes"
then
	torture_bootargs="torture.disable_onoff_at_boot"
	torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "14*CFLIST" --trust-make
fi

if test "$do_scftorture" = "yes"
then
	torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot"
	torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
fi

if test "$do_refscale" = yes
then
	primlist="`grep '\.name[ 	]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
else
	primlist=
fi
for prim in $primlist
do
	torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot"
	torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "verbose_batched=$VERBOSE_BATCH_CPUS" --trust-make
done

if test "$do_rcuscale" = yes
then
	primlist="`grep '\.name[ 	]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
else
	primlist=
fi
for prim in $primlist
do
	torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot"
	torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
done

if test "$do_kvfree" = "yes"
then
	torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot"
	torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
fi

echo " --- " $scriptname $args
echo " --- " Done `date` | tee -a $T/log
ret=0
nsuccesses=0
echo SUCCESSES: | tee -a $T/log
if test -s "$T/successes"
then
	cat "$T/successes" | tee -a $T/log
	nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`"
fi
nfailures=0
echo FAILURES: | tee -a $T/log
if test -s "$T/failures"
then
	cat "$T/failures" | tee -a $T/log
	nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`"
	ret=2
fi
echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log
echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log
tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`"
if test -n "$tdir"
then
	cp $T/log $tdir
fi
exit $ret

# @@@
# Need a way for the invoker to specify clang.  Maybe --kcsan-kmake or some such.
# --kconfig as with --bootargs (Both have overrides.)
# Command line parameters for --bootargs, --config, --kconfig, --kmake-arg, and --qemu-arg