Commit 2964f6b8 authored by Hangbin Liu's avatar Hangbin Liu Committed by Shuah Khan
Browse files

selftests: Use ktap helpers for runner.sh 



Instead of manually writing ktap messages, we should use the formal
ktap helpers in runner.sh. Brendan did some work in commit d9e6269e
("selftests/run_kselftest.sh: exit with error if tests fail") to make
run_kselftest.sh exit with the correct return value. However, the output
does not include the total results, such as how many tests passed or failed.

Let’s convert all manually printed messages in runner.sh to use the
formal ktap helpers. Here are what I changed:

  1. Move TAP header from runner.sh to run_kselftest.sh, since
     run_kselftest.sh is the only caller of run_many().
  2. In run_kselftest.sh, call run_many() in main process to count the
     pass/fail numbers.
  3. In run_kselftest.sh, do not generate kselftest_failures_file. Just
     use ktap_print_totals to report the result.
  4. In runner.sh run_one(), get the return value and use ktap helpers for
     all pass/fail reporting. This allows counting pass/fail numbers in the
     main process.
  5. In runner.sh run_in_netns(), also return the correct rc, so we can
     count results during wait.

After the change, the printed result looks like:

  not ok 4 4 selftests: clone3: clone3_cap_checkpoint_restore # exit=1
  # Totals: pass:3 fail:1 xfail:0 xpass:0 skip:0 error:0

  ]# echo $?
  1

Fixed change log commit description errors and long lines:
Shuah Khan <skhan@linuxfoundation.org>

Tested-by: default avatarBrendan Jackman <jackmanb@google.com>
Signed-off-by: default avatarHangbin Liu <liuhangbin@gmail.com>
Reviewed-by: default avatarBrendan Jackman <jackmanb@google.com>
Link: https://lore.kernel.org/r/20260225010833.11301-1-liuhangbin@gmail.com


Signed-off-by: default avatarShuah Khan <skhan@linuxfoundation.org>
parent 132618c5

tools/testing/selftests/kselftest/runner.sh

+60 −35
Original line number Diff line number Diff line 
#!/bin/sh

#!/bin/bash

# SPDX-License-Identifier: GPL-2.0

#

# Runs a set of tests in a given subdirectory.

export skip_rc=4

. $(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/ktap_helpers.sh

export timeout_rc=124

export logfile=/dev/stdout

export per_test_logging=
@@ -44,17 +44,11 @@ tap_timeout() 
	fi

}



report_failure()

{

	echo "not ok $*"

	echo "$*" >> "$kselftest_failures_file"

}



run_one()

{

	DIR="$1"

	TEST="$2"

	local test_num="$3"

	local rc test_num="$3"



	BASENAME_TEST=$(basename $TEST)
@@ -102,16 +96,17 @@ run_one() 
	# Command line timeout overrides the settings file

	if [ -n "$kselftest_override_timeout" ]; then

		kselftest_timeout="$kselftest_override_timeout"

		echo "# overriding timeout to $kselftest_timeout" >> "$logfile"

		ktap_print_msg "overriding timeout to $kselftest_timeout" >> "$logfile"

	else

		echo "# timeout set to $kselftest_timeout" >> "$logfile"

		ktap_print_msg "timeout set to $kselftest_timeout" >> "$logfile"

	fi



	TEST_HDR_MSG="selftests: $DIR: $BASENAME_TEST"

	echo "# $TEST_HDR_MSG"

	if [ ! -e "$TEST" ]; then

		echo "# Warning: file $TEST is missing!"

		report_failure "$test_num $TEST_HDR_MSG"

		ktap_print_msg "Warning: file $TEST is missing!"

		ktap_test_fail "$test_num $TEST_HDR_MSG"

		rc=$KSFT_FAIL

	else

		if [ -x /usr/bin/stdbuf ]; then

			stdbuf="/usr/bin/stdbuf --output=L "
@@ -122,33 +117,38 @@ run_one() 
		elif [ -x "./ksft_runner.sh" ]; then

			cmd="$stdbuf ./ksft_runner.sh ./$BASENAME_TEST"

		else

			echo "# Warning: file $TEST is not executable"

			ktap_print_msg "Warning: file $TEST is not executable"



			if [ $(head -n 1 "$TEST" | cut -c -2) = "#!" ]

			then

				interpreter=$(head -n 1 "$TEST" | cut -c 3-)

				cmd="$stdbuf $interpreter ./$BASENAME_TEST"

			else

				report_failure "$test_num $TEST_HDR_MSG"

				return

				ktap_test_fail "$test_num $TEST_HDR_MSG"

				return $KSFT_FAIL

			fi

		fi

		cd `dirname $TEST` > /dev/null

		((((( tap_timeout "$cmd" 2>&1; echo $? >&3) |

		(((( tap_timeout "$cmd" 2>&1; echo $? >&3) |

			tap_prefix >&4) 3>&1) |

			(read xs; exit $xs)) 4>>"$logfile" &&

		echo "ok $test_num $TEST_HDR_MSG") ||

		(rc=$?;	\

		if [ $rc -eq $skip_rc ]; then	\

			echo "ok $test_num $TEST_HDR_MSG # SKIP"

		elif [ $rc -eq $timeout_rc ]; then \

			echo "#"

			report_failure "$test_num $TEST_HDR_MSG # TIMEOUT $kselftest_timeout seconds"

		else

			report_failure "$test_num $TEST_HDR_MSG # exit=$rc"

		fi)

			(read xs; exit $xs)) 4>>"$logfile"

		rc=$?

		case "$rc" in

		"$KSFT_PASS")

			ktap_test_pass "$test_num $TEST_HDR_MSG";;

		"$KSFT_SKIP")

			ktap_test_skip "$test_num $TEST_HDR_MSG";;

		"$KSFT_XFAIL")

			ktap_test_xfail "$test_num $TEST_HDR_MSG";;

		"$timeout_rc")

			ktap_test_fail "$test_num $TEST_HDR_MSG # TIMEOUT $kselftest_timeout seconds";;

		*)

			ktap_test_fail "$test_num $TEST_HDR_MSG # exit=$rc";;

		esac

		cd - >/dev/null

	fi



	return $rc

}



in_netns()
@@ -164,27 +164,34 @@ in_netns() 


run_in_netns()

{

	local netns=$(mktemp -u ${BASENAME_TEST}-XXXXXX)

	local tmplog="/tmp/$(mktemp -u ${BASENAME_TEST}-XXXXXX)"

	local netns=$(mktemp -u ${BASENAME_TEST}-XXXXXX)

	local rc



	ip netns add $netns

	if [ $? -ne 0 ]; then

		echo "# Warning: Create namespace failed for $BASENAME_TEST"

		echo "not ok $test_num selftests: $DIR: $BASENAME_TEST # Create NS failed"

		ktap_print_msg "Warning: Create namespace failed for $BASENAME_TEST"

		ktap_test_fail "$test_num selftests: $DIR: $BASENAME_TEST # Create NS failed"

	fi

	ip -n $netns link set lo up



	in_netns $netns &> $tmplog

	rc=$?



	ip netns del $netns &> /dev/null

	# Cat the log at once to avoid parallel netns logs.

	cat $tmplog

	rm -f $tmplog

	return $rc

}



run_many()

{

	echo "TAP version 13"

	DIR="${PWD#${BASE_DIR}/}"

	test_num=0

	total=$(echo "$@" | wc -w)

	echo "1..$total"

	local rc

	pids=()



	for TEST in "$@"; do

		BASENAME_TEST=$(basename $TEST)

		test_num=$(( test_num + 1 ))
@@ -194,10 +201,28 @@ run_many() 
		fi

		if [ -n "$RUN_IN_NETNS" ]; then

			run_in_netns &

			pids+=($!)

		else

			run_one "$DIR" "$TEST" "$test_num"

		fi

	done



	wait

	# These variables are outputs of ktap_helpers.sh but since we've

	# run the test in a subprocess we need to update them manually

	for pid in "${pids[@]}"; do

		wait "$pid"

		rc=$?

		case "$rc" in

		"$KSFT_PASS")

			KTAP_CNT_PASS=$((KTAP_CNT_PASS + 1));;

		"$KSFT_FAIL")

			KTAP_CNT_FAIL=$((KTAP_CNT_FAIL + 1));;

		"$KSFT_SKIP")

			KTAP_CNT_SKIP=$((KTAP_CNT_SKIP + 1));;

		"$KSFT_XFAIL")

			KTAP_CNT_XFAIL=$((KTAP_CNT_XFAIL + 1));;

		*)

			KTAP_CNT_FAIL=$((KTAP_CNT_FAIL + 1));;

		esac

	done

}

tools/testing/selftests/run_kselftest.sh

+13 −8
Original line number Diff line number Diff line
@@ -121,18 +121,23 @@ if [ -n "$SKIP" ]; then 
	done

fi



kselftest_failures_file="$(mktemp --tmpdir kselftest-failures-XXXXXX)"

export kselftest_failures_file



curdir=$(pwd)

total=$(echo "$available" | wc -w)

collections=$(echo "$available" | cut -d: -f1 | sort | uniq)



ktap_print_header

ktap_set_plan "$total"



for collection in $collections ; do

	[ -w /dev/kmsg ] && echo "kselftest: Running tests in $collection" >> /dev/kmsg

	tests=$(echo "$available" | grep "^$collection:" | cut -d: -f2)

	($dryrun cd "$collection" && $dryrun run_many $tests)

	$dryrun cd "$collection" && $dryrun run_many $tests

	$dryrun cd "$curdir"

done



failures="$(cat "$kselftest_failures_file")"

rm "$kselftest_failures_file"

if "$ERROR_ON_FAIL" && [ "$failures" ]; then

	exit 1

ktap_print_totals

if "$ERROR_ON_FAIL" && [ "$KTAP_CNT_FAIL" -ne 0 ]; then

	exit "$KSFT_FAIL"

else

	exit "$KSFT_PASS"

fi