Commit d9e6269e authored by Brendan Jackman's avatar Brendan Jackman Committed by Shuah Khan
Browse files

selftests/run_kselftest.sh: exit with error if tests fail 

Parsing KTAP is quite an inconvenience, but most of the time the thing
you really want to know is "did anything fail"?

Let's give the user the his information without them needing
to parse anything.

Because of the use of subshells and namespaces, this needs to be
communicated via a file. Just write arbitrary data into the file and
treat non-empty content as a signal that something failed.

In case any user depends on the current behaviour, such as running this
from a script with `set -e` and parsing the result for failures
afterwards, add a flag they can set to get the old behaviour, namely
--no-error-on-fail.

Link: https://lore.kernel.org/r/20251111-b4-ksft-error-on-fail-v3-1-0951a51135f6@google.com


Signed-off-by: default avatarBrendan Jackman <jackmanb@google.com>
Signed-off-by: default avatarShuah Khan <skhan@linuxfoundation.org>
parent 26347f84

tools/testing/selftests/kselftest/runner.sh

+10 −4
Original line number Diff line number Diff line
@@ -44,6 +44,12 @@ tap_timeout() 
	fi

}



report_failure()

{

	echo "not ok $*"

	echo "$*" >> "$kselftest_failures_file"

}



run_one()

{

	DIR="$1"
@@ -105,7 +111,7 @@ run_one() 
	echo "# $TEST_HDR_MSG"

	if [ ! -e "$TEST" ]; then

		echo "# Warning: file $TEST is missing!"

		echo "not ok $test_num $TEST_HDR_MSG"

		report_failure "$test_num $TEST_HDR_MSG"

	else

		if [ -x /usr/bin/stdbuf ]; then

			stdbuf="/usr/bin/stdbuf --output=L "
@@ -123,7 +129,7 @@ run_one() 
				interpreter=$(head -n 1 "$TEST" | cut -c 3-)

				cmd="$stdbuf $interpreter ./$BASENAME_TEST"

			else

				echo "not ok $test_num $TEST_HDR_MSG"

				report_failure "$test_num $TEST_HDR_MSG"

				return

			fi

		fi
@@ -137,9 +143,9 @@ run_one() 
			echo "ok $test_num $TEST_HDR_MSG # SKIP"

		elif [ $rc -eq $timeout_rc ]; then \

			echo "#"

			echo "not ok $test_num $TEST_HDR_MSG # TIMEOUT $kselftest_timeout seconds"

			report_failure "$test_num $TEST_HDR_MSG # TIMEOUT $kselftest_timeout seconds"

		else

			echo "not ok $test_num $TEST_HDR_MSG # exit=$rc"

			report_failure "$test_num $TEST_HDR_MSG # exit=$rc"

		fi)

		cd - >/dev/null

	fi

tools/testing/selftests/run_kselftest.sh

+14 −0
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ Usage: $0 [OPTIONS] 
  -c | --collection COLLECTION	Run all tests from COLLECTION

  -l | --list			List the available collection:test entries

  -d | --dry-run		Don't actually run any tests

  -f | --no-error-on-fail	Don't exit with an error just because tests failed

  -n | --netns			Run each test in namespace

  -h | --help			Show this usage info

  -o | --override-timeout	Number of seconds after which we timeout
@@ -44,6 +45,7 @@ COLLECTIONS="" 
TESTS=""

dryrun=""

kselftest_override_timeout=""

ERROR_ON_FAIL=true

while true; do

	case "$1" in

		-s | --summary)
@@ -65,6 +67,9 @@ while true; do 
		-d | --dry-run)

			dryrun="echo"

			shift ;;

		-f | --no-error-on-fail)

			ERROR_ON_FAIL=false

			shift ;;

		-n | --netns)

			RUN_IN_NETNS=1

			shift ;;
@@ -105,9 +110,18 @@ if [ -n "$TESTS" ]; then 
	available="$(echo "$valid" | sed -e 's/ /\n/g')"

fi



kselftest_failures_file="$(mktemp --tmpdir kselftest-failures-XXXXXX)"

export kselftest_failures_file



collections=$(echo "$available" | cut -d: -f1 | sort | uniq)

for collection in $collections ; do

	[ -w /dev/kmsg ] && echo "kselftest: Running tests in $collection" >> /dev/kmsg

	tests=$(echo "$available" | grep "^$collection:" | cut -d: -f2)

	($dryrun cd "$collection" && $dryrun run_many $tests)

done



failures="$(cat "$kselftest_failures_file")"

rm "$kselftest_failures_file"

if "$ERROR_ON_FAIL" && [ "$failures" ]; then

	exit 1

fi