Commit d65d07cb authored by Rae Moar's avatar Rae Moar Committed by Shuah Khan
Browse files

kunit: tool: improve compatibility of kunit_parser with KTAP specification 

Update to kunit_parser to improve compatibility with KTAP
specification including arbitrarily nested tests. Patch accomplishes
three major changes:

- Use a general Test object to represent all tests rather than TestCase
and TestSuite objects. This allows for easier implementation of arbitrary
levels of nested tests and promotes the idea that both test suites and test
cases are tests.

- Print errors incrementally rather than all at once after the
parsing finishes to maximize information given to the user in the
case of the parser given invalid input and to increase the helpfulness
of the timestamps given during printing. Note that kunit.py parse does
not print incrementally yet. However, this fix brings us closer to
this feature.

- Increase compatibility for different formats of input. Arbitrary levels
of nested tests supported. Also, test cases and test suites are now
supported to be present on the same level of testing.

This patch now implements the draft KTAP specification here:
https://lore.kernel.org/linux-kselftest/CA+GJov6tdjvY9x12JsJT14qn6c7NViJxqaJk+r-K1YJzPggFDQ@mail.gmail.com/


We'll update the parser as the spec evolves.

This patch adjusts the kunit_tool_test.py file to check for
the correct outputs from the new parser and adds a new test to check
the parsing for a KTAP result log with correct format for multiple nested
subtests (test_is_test_passed-all_passed_nested.log).

This patch also alters the kunit_json.py file to allow for arbitrarily
nested tests.

Signed-off-by: default avatarRae Moar <rmoar@google.com>
Reviewed-by: default avatarBrendan Higgins <brendanhiggins@google.com>
Signed-off-by: default avatarDaniel Latypov <dlatypov@google.com>
Reviewed-by: default avatarDavid Gow <davidgow@google.com>
Signed-off-by: default avatarShuah Khan <skhan@linuxfoundation.org>
parent 7d7c48df

tools/testing/kunit/kunit.py

+15 −4
Original line number Diff line number Diff line
@@ -135,7 +135,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest, 
				test_glob = request.filter_glob.split('.', maxsplit=2)[1]

				filter_globs = [g + '.'+ test_glob for g in filter_globs]



	overall_status = kunit_parser.TestStatus.SUCCESS

	test_counts = kunit_parser.TestCounts()

	exec_time = 0.0

	for i, filter_glob in enumerate(filter_globs):

		kunit_parser.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs)))
@@ -154,18 +154,29 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest, 
		test_end = time.time()

		exec_time += test_end - test_start



		overall_status = kunit_parser.max_status(overall_status, result.status)

		test_counts.add_subtest_counts(result.result.test.counts)



	return KunitResult(status=result.status, result=result.result, elapsed_time=exec_time)

	kunit_status = _map_to_overall_status(test_counts.get_status())

	return KunitResult(status=kunit_status, result=result.result, elapsed_time=exec_time)



def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus:

	if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED):

		return KunitStatus.SUCCESS

	else:

		return KunitStatus.TEST_FAILURE



def parse_tests(request: KunitParseRequest, input_data: Iterable[str]) -> KunitResult:

	parse_start = time.time()



	test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS,

					      [],

					      kunit_parser.Test(),

					      'Tests not Parsed.')



	if request.raw_output:

		# Treat unparsed results as one passing test.

		test_result.test.status = kunit_parser.TestStatus.SUCCESS

		test_result.test.counts.passed = 1



		output: Iterable[str] = input_data

		if request.raw_output == 'all':

			pass

tools/testing/kunit/kunit_json.py

+28 −28
Original line number Diff line number Diff line
@@ -11,47 +11,47 @@ import os 


import kunit_parser



from kunit_parser import TestStatus

from kunit_parser import Test, TestResult, TestStatus

from typing import Any, Dict, Optional



def get_json_result(test_result, def_config, build_dir, json_path) -> str:

	sub_groups = []

JsonObj = Dict[str, Any]



	# Each test suite is mapped to a KernelCI sub_group

	for test_suite in test_result.suites:

		sub_group = {

			"name": test_suite.name,

			"arch": "UM",

			"defconfig": def_config,

			"build_environment": build_dir,

			"test_cases": [],

			"lab_name": None,

			"kernel": None,

			"job": None,

			"git_branch": "kselftest",

		}

		test_cases = []

		# TODO: Add attachments attribute in test_case with detailed

		#  failure message, see https://api.kernelci.org/schema-test-case.html#get

		for case in test_suite.cases:

			test_case = {"name": case.name, "status": "FAIL"}

			if case.status == TestStatus.SUCCESS:

def _get_group_json(test: Test, def_config: str,

		build_dir: Optional[str]) -> JsonObj:

	sub_groups = []  # List[JsonObj]

	test_cases = []  # List[JsonObj]



	for subtest in test.subtests:

		if len(subtest.subtests):

			sub_group = _get_group_json(subtest, def_config,

				build_dir)

			sub_groups.append(sub_group)

		else:

			test_case = {"name": subtest.name, "status": "FAIL"}

			if subtest.status == TestStatus.SUCCESS:

				test_case["status"] = "PASS"

			elif case.status == TestStatus.TEST_CRASHED:

			elif subtest.status == TestStatus.TEST_CRASHED:

				test_case["status"] = "ERROR"

			test_cases.append(test_case)

		sub_group["test_cases"] = test_cases

		sub_groups.append(sub_group)



	test_group = {

		"name": "KUnit Test Group",

		"name": test.name,

		"arch": "UM",

		"defconfig": def_config,

		"build_environment": build_dir,

		"sub_groups": sub_groups,

		"test_cases": test_cases,

		"lab_name": None,

		"kernel": None,

		"job": None,

		"git_branch": "kselftest",

	}

	return test_group



def get_json_result(test_result: TestResult, def_config: str,

		build_dir: Optional[str], json_path: str) -> str:

	test_group = _get_group_json(test_result.test, def_config, build_dir)

	test_group["name"] = "KUnit Test Group"

	json_obj = json.dumps(test_group, indent=4)

	if json_path != 'stdout':

		with open(json_path, 'w') as result_path:

tools/testing/kunit/kunit_parser.py

+702 −313

File changed.

Preview size limit exceeded, changes collapsed.

tools/testing/kunit/kunit_tool_test.py

+98 −38
Original line number Diff line number Diff line
@@ -150,6 +150,22 @@ class KUnitParserTest(unittest.TestCase): 
			kunit_parser.TestStatus.SUCCESS,

			result.status)



	def test_parse_successful_nested_tests_log(self):

		all_passed_log = test_data_path('test_is_test_passed-all_passed_nested.log')

		with open(all_passed_log) as file:

			result = kunit_parser.parse_run_tests(file.readlines())

		self.assertEqual(

			kunit_parser.TestStatus.SUCCESS,

			result.status)



	def test_kselftest_nested(self):

		kselftest_log = test_data_path('test_is_test_passed-kselftest.log')

		with open(kselftest_log) as file:

			result = kunit_parser.parse_run_tests(file.readlines())

			self.assertEqual(

				kunit_parser.TestStatus.SUCCESS,

				result.status)



	def test_parse_failed_test_log(self):

		failed_log = test_data_path('test_is_test_passed-failure.log')

		with open(failed_log) as file:
@@ -163,17 +179,29 @@ class KUnitParserTest(unittest.TestCase): 
		with open(empty_log) as file:

			result = kunit_parser.parse_run_tests(

				kunit_parser.extract_tap_lines(file.readlines()))

		self.assertEqual(0, len(result.suites))

		self.assertEqual(0, len(result.test.subtests))

		self.assertEqual(

			kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS,

			result.status)



	def test_missing_test_plan(self):

		missing_plan_log = test_data_path('test_is_test_passed-'

			'missing_plan.log')

		with open(missing_plan_log) as file:

			result = kunit_parser.parse_run_tests(

				kunit_parser.extract_tap_lines(

				file.readlines()))

		self.assertEqual(2, result.test.counts.errors)

		self.assertEqual(

			kunit_parser.TestStatus.SUCCESS,

			result.status)



	def test_no_tests(self):

		empty_log = test_data_path('test_is_test_passed-no_tests_run_with_header.log')

		with open(empty_log) as file:

		header_log = test_data_path('test_is_test_passed-no_tests_run_with_header.log')

		with open(header_log) as file:

			result = kunit_parser.parse_run_tests(

				kunit_parser.extract_tap_lines(file.readlines()))

		self.assertEqual(0, len(result.suites))

		self.assertEqual(0, len(result.test.subtests))

		self.assertEqual(

			kunit_parser.TestStatus.NO_TESTS,

			result.status)
@@ -184,14 +212,15 @@ class KUnitParserTest(unittest.TestCase): 
		with open(crash_log) as file:

			result = kunit_parser.parse_run_tests(

				kunit_parser.extract_tap_lines(file.readlines()))

		print_mock.assert_any_call(StrContains('could not parse test results!'))

		print_mock.assert_any_call(StrContains('invalid KTAP input!'))

		print_mock.stop()

		self.assertEqual(0, len(result.suites))

		self.assertEqual(0, len(result.test.subtests))



	def test_crashed_test(self):

		crashed_log = test_data_path('test_is_test_passed-crash.log')

		with open(crashed_log) as file:

			result = kunit_parser.parse_run_tests(file.readlines())

			result = kunit_parser.parse_run_tests(

				file.readlines())

		self.assertEqual(

			kunit_parser.TestStatus.TEST_CRASHED,

			result.status)
@@ -215,6 +244,23 @@ class KUnitParserTest(unittest.TestCase): 
			kunit_parser.TestStatus.SKIPPED,

			result.status)



	def test_ignores_hyphen(self):

		hyphen_log = test_data_path('test_strip_hyphen.log')

		file = open(hyphen_log)

		result = kunit_parser.parse_run_tests(file.readlines())



		# A skipped test does not fail the whole suite.

		self.assertEqual(

			kunit_parser.TestStatus.SUCCESS,

			result.status)

		self.assertEqual(

			"sysctl_test",

			result.test.subtests[0].name)

		self.assertEqual(

			"example",

			result.test.subtests[1].name)

		file.close()





	def test_ignores_prefix_printk_time(self):

		prefix_log = test_data_path('test_config_printk_time.log')
@@ -223,7 +269,7 @@ class KUnitParserTest(unittest.TestCase): 
			self.assertEqual(

				kunit_parser.TestStatus.SUCCESS,

				result.status)

			self.assertEqual('kunit-resource-test', result.suites[0].name)

			self.assertEqual('kunit-resource-test', result.test.subtests[0].name)



	def test_ignores_multiple_prefixes(self):

		prefix_log = test_data_path('test_multiple_prefixes.log')
@@ -232,7 +278,7 @@ class KUnitParserTest(unittest.TestCase): 
			self.assertEqual(

				kunit_parser.TestStatus.SUCCESS,

				result.status)

			self.assertEqual('kunit-resource-test', result.suites[0].name)

			self.assertEqual('kunit-resource-test', result.test.subtests[0].name)



	def test_prefix_mixed_kernel_output(self):

		mixed_prefix_log = test_data_path('test_interrupted_tap_output.log')
@@ -241,7 +287,7 @@ class KUnitParserTest(unittest.TestCase): 
			self.assertEqual(

				kunit_parser.TestStatus.SUCCESS,

				result.status)

			self.assertEqual('kunit-resource-test', result.suites[0].name)

			self.assertEqual('kunit-resource-test', result.test.subtests[0].name)



	def test_prefix_poundsign(self):

		pound_log = test_data_path('test_pound_sign.log')
@@ -250,7 +296,7 @@ class KUnitParserTest(unittest.TestCase): 
			self.assertEqual(

				kunit_parser.TestStatus.SUCCESS,

				result.status)

			self.assertEqual('kunit-resource-test', result.suites[0].name)

			self.assertEqual('kunit-resource-test', result.test.subtests[0].name)



	def test_kernel_panic_end(self):

		panic_log = test_data_path('test_kernel_panic_interrupt.log')
@@ -259,7 +305,7 @@ class KUnitParserTest(unittest.TestCase): 
			self.assertEqual(

				kunit_parser.TestStatus.TEST_CRASHED,

				result.status)

			self.assertEqual('kunit-resource-test', result.suites[0].name)

			self.assertEqual('kunit-resource-test', result.test.subtests[0].name)



	def test_pound_no_prefix(self):

		pound_log = test_data_path('test_pound_no_prefix.log')
@@ -268,7 +314,7 @@ class KUnitParserTest(unittest.TestCase): 
			self.assertEqual(

				kunit_parser.TestStatus.SUCCESS,

				result.status)

			self.assertEqual('kunit-resource-test', result.suites[0].name)

			self.assertEqual('kunit-resource-test', result.test.subtests[0].name)



class LinuxSourceTreeTest(unittest.TestCase):
@@ -341,6 +387,12 @@ class KUnitJsonTest(unittest.TestCase): 
		result = self._json_for('test_is_test_passed-no_tests_run_with_header.log')

		self.assertEqual(0, len(result['sub_groups']))



	def test_nested_json(self):

		result = self._json_for('test_is_test_passed-all_passed_nested.log')

		self.assertEqual(

			{'name': 'example_simple_test', 'status': 'PASS'},

			result["sub_groups"][0]["sub_groups"][0]["test_cases"][0])



class StrContains(str):

	def __eq__(self, other):

		return self in other
@@ -399,7 +451,15 @@ class KUnitMainTest(unittest.TestCase): 
		self.assertEqual(e.exception.code, 1)

		self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)

		self.assertEqual(self.linux_source_mock.run_kernel.call_count, 1)

		self.print_mock.assert_any_call(StrContains(' 0 tests run'))

		self.print_mock.assert_any_call(StrContains('invalid KTAP input!'))



	def test_exec_no_tests(self):

		self.linux_source_mock.run_kernel = mock.Mock(return_value=['TAP version 14', '1..0'])

		with self.assertRaises(SystemExit) as e:

                  kunit.main(['run'], self.linux_source_mock)

		self.linux_source_mock.run_kernel.assert_called_once_with(

			args=None, build_dir='.kunit', filter_glob='', timeout=300)

		self.print_mock.assert_any_call(StrContains(' 0 tests run!'))



	def test_exec_raw_output(self):

		self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
@@ -407,7 +467,7 @@ class KUnitMainTest(unittest.TestCase): 
		self.assertEqual(self.linux_source_mock.run_kernel.call_count, 1)

		for call in self.print_mock.call_args_list:

			self.assertNotEqual(call, mock.call(StrContains('Testing complete.')))

			self.assertNotEqual(call, mock.call(StrContains(' 0 tests run')))

			self.assertNotEqual(call, mock.call(StrContains(' 0 tests run!')))



	def test_run_raw_output(self):

		self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
@@ -416,7 +476,7 @@ class KUnitMainTest(unittest.TestCase): 
		self.assertEqual(self.linux_source_mock.run_kernel.call_count, 1)

		for call in self.print_mock.call_args_list:

			self.assertNotEqual(call, mock.call(StrContains('Testing complete.')))

			self.assertNotEqual(call, mock.call(StrContains(' 0 tests run')))

			self.assertNotEqual(call, mock.call(StrContains(' 0 tests run!')))



	def test_run_raw_output_kunit(self):

		self.linux_source_mock.run_kernel = mock.Mock(return_value=[])

tools/testing/kunit/test_data/test_is_test_passed-all_passed_nested.log

0 → 100644
+34 −0
Original line number Diff line number Diff line 
TAP version 14

1..2

	# Subtest: sysctl_test

	1..4

	# sysctl_test_dointvec_null_tbl_data: sysctl_test_dointvec_null_tbl_data passed

	ok 1 - sysctl_test_dointvec_null_tbl_data

		# Subtest: example

		1..2

	init_suite

		# example_simple_test: initializing

		# example_simple_test: example_simple_test passed

		ok 1 - example_simple_test

		# example_mock_test: initializing

		# example_mock_test: example_mock_test passed

		ok 2 - example_mock_test

	kunit example: all tests passed

	ok 2 - example

	# sysctl_test_dointvec_table_len_is_zero: sysctl_test_dointvec_table_len_is_zero passed

	ok 3 - sysctl_test_dointvec_table_len_is_zero

	# sysctl_test_dointvec_table_read_but_position_set: sysctl_test_dointvec_table_read_but_position_set passed

	ok 4 - sysctl_test_dointvec_table_read_but_position_set

kunit sysctl_test: all tests passed

ok 1 - sysctl_test

	# Subtest: example

	1..2

init_suite

	# example_simple_test: initializing

	# example_simple_test: example_simple_test passed

	ok 1 - example_simple_test

	# example_mock_test: initializing

	# example_mock_test: example_mock_test passed

	ok 2 - example_mock_test

kunit example: all tests passed

ok 2 - example