Commit 313b6ffc authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'linux-kselftest-kunit-5.16-rc1' of... 

Merge tag 'linux-kselftest-kunit-5.16-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest

Pull KUnit updates from Shuah Khan:
 "Several enhancements and fixes:

   - ability to run each test suite and test separately

   - support for timing test run

   - several fixes and improvements"

* tag 'linux-kselftest-kunit-5.16-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest:
  kunit: tool: fix typecheck errors about loading qemu configs
  kunit: tool: continue past invalid utf-8 output
  kunit: Reset suite count after running tests
  kunit: tool: improve compatibility of kunit_parser with KTAP specification
  kunit: tool: yield output from run_kernel in real time
  kunit: tool: support running each suite/test separately
  kunit: tool: actually track how long it took to run tests
  kunit: tool: factor exec + parse steps into a function
  kunit: add 'kunit.action' param to allow listing out tests
  kunit: tool: show list of valid --arch options when invalid
  kunit: tool: misc fixes (unused vars, imports, leaked files)
  kunit: fix too small allocation when using suite-only kunit.filter_glob
  kunit: tool: allow filtering test cases via glob
  kunit: drop assumption in kunit-log-test about current suite
parents 84924e2e 52a5d80a

Documentation/dev-tools/kunit/running_tips.rst

+9 −2
Original line number Diff line number Diff line
@@ -25,8 +25,8 @@ It can be handy to create a bash function like: 
Running a subset of tests

-------------------------



``kunit.py run`` accepts an optional glob argument to filter tests. Currently

this only matches against suite names, but this may change in the future.

``kunit.py run`` accepts an optional glob argument to filter tests. The format

is ``"<suite_glob>[.test_glob]"``.



Say that we wanted to run the sysctl tests, we could do so via:
@@ -35,6 +35,13 @@ Say that we wanted to run the sysctl tests, we could do so via: 
	$ echo -e 'CONFIG_KUNIT=y\nCONFIG_KUNIT_ALL_TESTS=y' > .kunit/.kunitconfig

	$ ./tools/testing/kunit/kunit.py run 'sysctl*'



We can filter down to just the "write" tests via:



.. code-block:: bash



	$ echo -e 'CONFIG_KUNIT=y\nCONFIG_KUNIT_ALL_TESTS=y' > .kunit/.kunitconfig

	$ ./tools/testing/kunit/kunit.py run 'sysctl*.*write*'



We're paying the cost of building more tests than we need this way, but it's

easier than fiddling with ``.kunitconfig`` files or commenting out

``kunit_suite``'s.

lib/kunit/executor.c

+136 −16
Original line number Diff line number Diff line
@@ -15,23 +15,89 @@ extern struct kunit_suite * const * const __kunit_suites_end[]; 
#if IS_BUILTIN(CONFIG_KUNIT)



static char *filter_glob_param;

static char *action_param;



module_param_named(filter_glob, filter_glob_param, charp, 0);

MODULE_PARM_DESC(filter_glob,

		"Filter which KUnit test suites run at boot-time, e.g. list*");

		"Filter which KUnit test suites/tests run at boot-time, e.g. list* or list*.*del_test");

module_param_named(action, action_param, charp, 0);

MODULE_PARM_DESC(action,

		 "Changes KUnit executor behavior, valid values are:\n"

		 "<none>: run the tests like normal\n"

		 "'list' to list test names instead of running them.\n");



/* glob_match() needs NULL terminated strings, so we need a copy of filter_glob_param. */

struct kunit_test_filter {

	char *suite_glob;

	char *test_glob;

};



/* Split "suite_glob.test_glob" into two. Assumes filter_glob is not empty. */

static void kunit_parse_filter_glob(struct kunit_test_filter *parsed,

				    const char *filter_glob)

{

	const int len = strlen(filter_glob);

	const char *period = strchr(filter_glob, '.');



	if (!period) {

		parsed->suite_glob = kzalloc(len + 1, GFP_KERNEL);

		parsed->test_glob = NULL;

		strcpy(parsed->suite_glob, filter_glob);

		return;

	}



	parsed->suite_glob = kzalloc(period - filter_glob + 1, GFP_KERNEL);

	parsed->test_glob = kzalloc(len - (period - filter_glob) + 1, GFP_KERNEL);



	strncpy(parsed->suite_glob, filter_glob, period - filter_glob);

	strncpy(parsed->test_glob, period + 1, len - (period - filter_glob));

}



/* Create a copy of suite with only tests that match test_glob. */

static struct kunit_suite *

kunit_filter_tests(struct kunit_suite *const suite, const char *test_glob)

{

	int n = 0;

	struct kunit_case *filtered, *test_case;

	struct kunit_suite *copy;



	kunit_suite_for_each_test_case(suite, test_case) {

		if (!test_glob || glob_match(test_glob, test_case->name))

			++n;

	}



	if (n == 0)

		return NULL;



	/* Use memcpy to workaround copy->name being const. */

	copy = kmalloc(sizeof(*copy), GFP_KERNEL);

	memcpy(copy, suite, sizeof(*copy));



	filtered = kcalloc(n + 1, sizeof(*filtered), GFP_KERNEL);



	n = 0;

	kunit_suite_for_each_test_case(suite, test_case) {

		if (!test_glob || glob_match(test_glob, test_case->name))

			filtered[n++] = *test_case;

	}



	copy->test_cases = filtered;

	return copy;

}



static char *kunit_shutdown;

core_param(kunit_shutdown, kunit_shutdown, charp, 0644);



static struct kunit_suite * const *

kunit_filter_subsuite(struct kunit_suite * const * const subsuite,

			const char *filter_glob)

		      struct kunit_test_filter *filter)

{

	int i, n = 0;

	struct kunit_suite **filtered;

	struct kunit_suite **filtered, *filtered_suite;



	n = 0;

	for (i = 0; subsuite[i] != NULL; ++i) {

		if (glob_match(filter_glob, subsuite[i]->name))

	for (i = 0; subsuite[i]; ++i) {

		if (glob_match(filter->suite_glob, subsuite[i]->name))

			++n;

	}
@@ -44,8 +110,11 @@ kunit_filter_subsuite(struct kunit_suite * const * const subsuite, 


	n = 0;

	for (i = 0; subsuite[i] != NULL; ++i) {

		if (glob_match(filter_glob, subsuite[i]->name))

			filtered[n++] = subsuite[i];

		if (!glob_match(filter->suite_glob, subsuite[i]->name))

			continue;

		filtered_suite = kunit_filter_tests(subsuite[i], filter->test_glob);

		if (filtered_suite)

			filtered[n++] = filtered_suite;

	}

	filtered[n] = NULL;
@@ -57,12 +126,32 @@ struct suite_set { 
	struct kunit_suite * const * const *end;

};



static void kunit_free_subsuite(struct kunit_suite * const *subsuite)

{

	unsigned int i;



	for (i = 0; subsuite[i]; i++)

		kfree(subsuite[i]);



	kfree(subsuite);

}



static void kunit_free_suite_set(struct suite_set suite_set)

{

	struct kunit_suite * const * const *suites;



	for (suites = suite_set.start; suites < suite_set.end; suites++)

		kunit_free_subsuite(*suites);

	kfree(suite_set.start);

}



static struct suite_set kunit_filter_suites(const struct suite_set *suite_set,

					    const char *filter_glob)

{

	int i;

	struct kunit_suite * const **copy, * const *filtered_subsuite;

	struct suite_set filtered;

	struct kunit_test_filter filter;



	const size_t max = suite_set->end - suite_set->start;
@@ -73,12 +162,17 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, 
		return filtered;

	}



	kunit_parse_filter_glob(&filter, filter_glob);



	for (i = 0; i < max; ++i) {

		filtered_subsuite = kunit_filter_subsuite(suite_set->start[i], filter_glob);

		filtered_subsuite = kunit_filter_subsuite(suite_set->start[i], &filter);

		if (filtered_subsuite)

			*copy++ = filtered_subsuite;

	}

	filtered.end = copy;



	kfree(filter.suite_glob);

	kfree(filter.test_glob);

	return filtered;

}
@@ -109,9 +203,35 @@ static void kunit_print_tap_header(struct suite_set *suite_set) 
	pr_info("1..%d\n", num_of_suites);

}



int kunit_run_all_tests(void)

static void kunit_exec_run_tests(struct suite_set *suite_set)

{

	struct kunit_suite * const * const *suites;



	kunit_print_tap_header(suite_set);



	for (suites = suite_set->start; suites < suite_set->end; suites++)

		__kunit_test_suites_init(*suites);

}



static void kunit_exec_list_tests(struct suite_set *suite_set)

{

	unsigned int i;

	struct kunit_suite * const * const *suites;

	struct kunit_case *test_case;



	/* Hack: print a tap header so kunit.py can find the start of KUnit output. */

	pr_info("TAP version 14\n");



	for (suites = suite_set->start; suites < suite_set->end; suites++)

		for (i = 0; (*suites)[i] != NULL; i++) {

			kunit_suite_for_each_test_case((*suites)[i], test_case) {

				pr_info("%s.%s\n", (*suites)[i]->name, test_case->name);

			}

		}

}



int kunit_run_all_tests(void)

{

	struct suite_set suite_set = {

		.start = __kunit_suites_start,

		.end = __kunit_suites_end,
@@ -120,15 +240,15 @@ int kunit_run_all_tests(void) 
	if (filter_glob_param)

		suite_set = kunit_filter_suites(&suite_set, filter_glob_param);



	kunit_print_tap_header(&suite_set);



	for (suites = suite_set.start; suites < suite_set.end; suites++)

		__kunit_test_suites_init(*suites);

	if (!action_param)

		kunit_exec_run_tests(&suite_set);

	else if (strcmp(action_param, "list") == 0)

		kunit_exec_list_tests(&suite_set);

	else

		pr_err("kunit executor: unknown action '%s'\n", action_param);



	if (filter_glob_param) { /* a copy was made of each array */

		for (suites = suite_set.start; suites < suite_set.end; suites++)

			kfree(*suites);

		kfree(suite_set.start);

		kunit_free_suite_set(suite_set);

	}



	kunit_handle_shutdown();

lib/kunit/executor_test.c

+97 −13
Original line number Diff line number Diff line
@@ -9,38 +9,103 @@ 
#include <kunit/test.h>



static void kfree_at_end(struct kunit *test, const void *to_free);

static void free_subsuite_at_end(struct kunit *test,

				 struct kunit_suite *const *to_free);

static struct kunit_suite *alloc_fake_suite(struct kunit *test,

					    const char *suite_name);

					    const char *suite_name,

					    struct kunit_case *test_cases);



static void dummy_test(struct kunit *test) {}



static struct kunit_case dummy_test_cases[] = {

	/* .run_case is not important, just needs to be non-NULL */

	{ .name = "test1", .run_case = dummy_test },

	{ .name = "test2", .run_case = dummy_test },

	{},

};



static void parse_filter_test(struct kunit *test)

{

	struct kunit_test_filter filter = {NULL, NULL};



	kunit_parse_filter_glob(&filter, "suite");

	KUNIT_EXPECT_STREQ(test, filter.suite_glob, "suite");

	KUNIT_EXPECT_FALSE(test, filter.test_glob);

	kfree(filter.suite_glob);

	kfree(filter.test_glob);



	kunit_parse_filter_glob(&filter, "suite.test");

	KUNIT_EXPECT_STREQ(test, filter.suite_glob, "suite");

	KUNIT_EXPECT_STREQ(test, filter.test_glob, "test");

	kfree(filter.suite_glob);

	kfree(filter.test_glob);

}



static void filter_subsuite_test(struct kunit *test)

{

	struct kunit_suite *subsuite[3] = {NULL, NULL, NULL};

	struct kunit_suite * const *filtered;

	struct kunit_test_filter filter = {

		.suite_glob = "suite2",

		.test_glob = NULL,

	};



	subsuite[0] = alloc_fake_suite(test, "suite1");

	subsuite[1] = alloc_fake_suite(test, "suite2");

	subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases);

	subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);



	/* Want: suite1, suite2, NULL -> suite2, NULL */

	filtered = kunit_filter_subsuite(subsuite, "suite2*");

	filtered = kunit_filter_subsuite(subsuite, &filter);

	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered);

	kfree_at_end(test, filtered);

	free_subsuite_at_end(test, filtered);



	/* Validate we just have suite2 */

	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]);

	KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->name, "suite2");

	KUNIT_EXPECT_FALSE(test, filtered[1]);

}



static void filter_subsuite_test_glob_test(struct kunit *test)

{

	struct kunit_suite *subsuite[3] = {NULL, NULL, NULL};

	struct kunit_suite * const *filtered;

	struct kunit_test_filter filter = {

		.suite_glob = "suite2",

		.test_glob = "test2",

	};



	subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases);

	subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);



	/* Want: suite1, suite2, NULL -> suite2 (just test1), NULL */

	filtered = kunit_filter_subsuite(subsuite, &filter);

	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered);

	free_subsuite_at_end(test, filtered);



	/* Validate we just have suite2 */

	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]);

	KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->name, "suite2");

	KUNIT_EXPECT_FALSE(test, filtered[1]);



	/* Now validate we just have test2 */

	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]->test_cases);

	KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->test_cases[0].name, "test2");

	KUNIT_EXPECT_FALSE(test, filtered[0]->test_cases[1].name);

}



static void filter_subsuite_to_empty_test(struct kunit *test)

{

	struct kunit_suite *subsuite[3] = {NULL, NULL, NULL};

	struct kunit_suite * const *filtered;

	struct kunit_test_filter filter = {

		.suite_glob = "not_found",

		.test_glob = NULL,

	};



	subsuite[0] = alloc_fake_suite(test, "suite1");

	subsuite[1] = alloc_fake_suite(test, "suite2");

	subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases);

	subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);



	filtered = kunit_filter_subsuite(subsuite, "not_found");

	kfree_at_end(test, filtered); /* just in case */

	filtered = kunit_filter_subsuite(subsuite, &filter);

	free_subsuite_at_end(test, filtered); /* just in case */



	KUNIT_EXPECT_FALSE_MSG(test, filtered,

			       "should be NULL to indicate no match");
@@ -52,7 +117,7 @@ static void kfree_subsuites_at_end(struct kunit *test, struct suite_set *suite_s 


	kfree_at_end(test, suite_set->start);

	for (suites = suite_set->start; suites < suite_set->end; suites++)

		kfree_at_end(test, *suites);

		free_subsuite_at_end(test, *suites);

}



static void filter_suites_test(struct kunit *test)
@@ -74,8 +139,8 @@ static void filter_suites_test(struct kunit *test) 
	struct suite_set filtered = {.start = NULL, .end = NULL};



	/* Emulate two files, each having one suite */

	subsuites[0][0] = alloc_fake_suite(test, "suite0");

	subsuites[1][0] = alloc_fake_suite(test, "suite1");

	subsuites[0][0] = alloc_fake_suite(test, "suite0", dummy_test_cases);

	subsuites[1][0] = alloc_fake_suite(test, "suite1", dummy_test_cases);



	/* Filter out suite1 */

	filtered = kunit_filter_suites(&suite_set, "suite0");
@@ -84,11 +149,14 @@ static void filter_suites_test(struct kunit *test) 


	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start);

	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start[0]);

	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start[0][0]);

	KUNIT_EXPECT_STREQ(test, (const char *)filtered.start[0][0]->name, "suite0");

}



static struct kunit_case executor_test_cases[] = {

	KUNIT_CASE(parse_filter_test),

	KUNIT_CASE(filter_subsuite_test),

	KUNIT_CASE(filter_subsuite_test_glob_test),

	KUNIT_CASE(filter_subsuite_to_empty_test),

	KUNIT_CASE(filter_suites_test),

	{}
@@ -120,14 +188,30 @@ static void kfree_at_end(struct kunit *test, const void *to_free) 
			     (void *)to_free);

}



static void free_subsuite_res_free(struct kunit_resource *res)

{

	kunit_free_subsuite(res->data);

}



static void free_subsuite_at_end(struct kunit *test,

				 struct kunit_suite *const *to_free)

{

	if (IS_ERR_OR_NULL(to_free))

		return;

	kunit_alloc_resource(test, NULL, free_subsuite_res_free,

			     GFP_KERNEL, (void *)to_free);

}



static struct kunit_suite *alloc_fake_suite(struct kunit *test,

					    const char *suite_name)

					    const char *suite_name,

					    struct kunit_case *test_cases)

{

	struct kunit_suite *suite;



	/* We normally never expect to allocate suites, hence the non-const cast. */

	suite = kunit_kzalloc(test, sizeof(*suite), GFP_KERNEL);

	strncpy((char *)suite->name, suite_name, sizeof(suite->name) - 1);

	suite->test_cases = test_cases;



	return suite;

}

lib/kunit/kunit-test.c

+8 −6
Original line number Diff line number Diff line
@@ -415,12 +415,15 @@ static struct kunit_suite kunit_log_test_suite = { 


static void kunit_log_test(struct kunit *test)

{

	struct kunit_suite *suite = &kunit_log_test_suite;

	struct kunit_suite suite;



	suite.log = kunit_kzalloc(test, KUNIT_LOG_SIZE, GFP_KERNEL);

	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);



	kunit_log(KERN_INFO, test, "put this in log.");

	kunit_log(KERN_INFO, test, "this too.");

	kunit_log(KERN_INFO, suite, "add to suite log.");

	kunit_log(KERN_INFO, suite, "along with this.");

	kunit_log(KERN_INFO, &suite, "add to suite log.");

	kunit_log(KERN_INFO, &suite, "along with this.");



#ifdef CONFIG_KUNIT_DEBUGFS

	KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
@@ -428,12 +431,11 @@ static void kunit_log_test(struct kunit *test) 
	KUNIT_EXPECT_NOT_ERR_OR_NULL(test,

				     strstr(test->log, "this too."));

	KUNIT_EXPECT_NOT_ERR_OR_NULL(test,

				     strstr(suite->log, "add to suite log."));

				     strstr(suite.log, "add to suite log."));

	KUNIT_EXPECT_NOT_ERR_OR_NULL(test,

				     strstr(suite->log, "along with this."));

				     strstr(suite.log, "along with this."));

#else

	KUNIT_EXPECT_PTR_EQ(test, test->log, (char *)NULL);

	KUNIT_EXPECT_PTR_EQ(test, suite->log, (char *)NULL);

#endif

}

lib/kunit/test.c

+4 −2
Original line number Diff line number Diff line
@@ -190,10 +190,10 @@ enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite) 
}

EXPORT_SYMBOL_GPL(kunit_suite_has_succeeded);



static void kunit_print_subtest_end(struct kunit_suite *suite)

{

static size_t kunit_suite_counter = 1;



static void kunit_print_subtest_end(struct kunit_suite *suite)

{

	kunit_print_ok_not_ok((void *)suite, false,

			      kunit_suite_has_succeeded(suite),

			      kunit_suite_counter++,
@@ -583,6 +583,8 @@ void __kunit_test_suites_exit(struct kunit_suite **suites) 


	for (i = 0; suites[i] != NULL; i++)

		kunit_exit_suite(suites[i]);



	kunit_suite_counter = 1;

}

EXPORT_SYMBOL_GPL(__kunit_test_suites_exit);