From a127b154a8f231709754b5d56a501163dd837459 Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Tue, 14 Sep 2021 14:03:48 -0700 Subject: kunit: tool: allow filtering test cases via glob Commit 1d71307a6f94 ("kunit: add unit test for filtering suites by names") introduced the ability to filter which suites we run via glob. This change extends it so we can also filter individual test cases inside of suites as well. This is quite useful when, e.g. * trying to run just the tests cases you've just added or are working on * trying to debug issues with test hermeticity Examples: $ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit '*exec*.parse*' ... ============================================================ ======== [PASSED] kunit_executor_test ======== [PASSED] parse_filter_test ============================================================ Testing complete. 1 tests run. 0 failed. 0 crashed. $ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit '*.no_matching_tests' ... [ERROR] no tests run! Signed-off-by: Daniel Latypov Reviewed-by: David Gow Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/kunit/executor.c | 107 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 96 insertions(+), 11 deletions(-) (limited to 'lib/kunit/executor.c') diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index acd1de436f59..bab3ab940acc 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -17,21 +17,80 @@ extern struct kunit_suite * const * const __kunit_suites_end[]; static char *filter_glob_param; module_param_named(filter_glob, filter_glob_param, charp, 0); MODULE_PARM_DESC(filter_glob, - "Filter which KUnit test suites run at boot-time, e.g. list*"); + "Filter which KUnit test suites/tests run at boot-time, e.g. list* or list*.*del_test"); + +/* glob_match() needs NULL terminated strings, so we need a copy of filter_glob_param. */ +struct kunit_test_filter { + char *suite_glob; + char *test_glob; +}; + +/* Split "suite_glob.test_glob" into two. Assumes filter_glob is not empty. */ +static void kunit_parse_filter_glob(struct kunit_test_filter *parsed, + const char *filter_glob) +{ + const int len = strlen(filter_glob); + const char *period = strchr(filter_glob, '.'); + + if (!period) { + parsed->suite_glob = kmalloc(len, GFP_KERNEL); + parsed->test_glob = NULL; + strcpy(parsed->suite_glob, filter_glob); + return; + } + + parsed->suite_glob = kzalloc(period - filter_glob + 1, GFP_KERNEL); + parsed->test_glob = kzalloc(len - (period - filter_glob) + 1, GFP_KERNEL); + + strncpy(parsed->suite_glob, filter_glob, period - filter_glob); + strncpy(parsed->test_glob, period + 1, len - (period - filter_glob)); +} + +/* Create a copy of suite with only tests that match test_glob. */ +static struct kunit_suite * +kunit_filter_tests(struct kunit_suite *const suite, const char *test_glob) +{ + int n = 0; + struct kunit_case *filtered, *test_case; + struct kunit_suite *copy; + + kunit_suite_for_each_test_case(suite, test_case) { + if (!test_glob || glob_match(test_glob, test_case->name)) + ++n; + } + + if (n == 0) + return NULL; + + /* Use memcpy to workaround copy->name being const. */ + copy = kmalloc(sizeof(*copy), GFP_KERNEL); + memcpy(copy, suite, sizeof(*copy)); + + filtered = kcalloc(n + 1, sizeof(*filtered), GFP_KERNEL); + + n = 0; + kunit_suite_for_each_test_case(suite, test_case) { + if (!test_glob || glob_match(test_glob, test_case->name)) + filtered[n++] = *test_case; + } + + copy->test_cases = filtered; + return copy; +} static char *kunit_shutdown; core_param(kunit_shutdown, kunit_shutdown, charp, 0644); static struct kunit_suite * const * kunit_filter_subsuite(struct kunit_suite * const * const subsuite, - const char *filter_glob) + struct kunit_test_filter *filter) { int i, n = 0; - struct kunit_suite **filtered; + struct kunit_suite **filtered, *filtered_suite; n = 0; - for (i = 0; subsuite[i] != NULL; ++i) { - if (glob_match(filter_glob, subsuite[i]->name)) + for (i = 0; subsuite[i]; ++i) { + if (glob_match(filter->suite_glob, subsuite[i]->name)) ++n; } @@ -44,8 +103,11 @@ kunit_filter_subsuite(struct kunit_suite * const * const subsuite, n = 0; for (i = 0; subsuite[i] != NULL; ++i) { - if (glob_match(filter_glob, subsuite[i]->name)) - filtered[n++] = subsuite[i]; + if (!glob_match(filter->suite_glob, subsuite[i]->name)) + continue; + filtered_suite = kunit_filter_tests(subsuite[i], filter->test_glob); + if (filtered_suite) + filtered[n++] = filtered_suite; } filtered[n] = NULL; @@ -57,12 +119,32 @@ struct suite_set { struct kunit_suite * const * const *end; }; +static void kunit_free_subsuite(struct kunit_suite * const *subsuite) +{ + unsigned int i; + + for (i = 0; subsuite[i]; i++) + kfree(subsuite[i]); + + kfree(subsuite); +} + +static void kunit_free_suite_set(struct suite_set suite_set) +{ + struct kunit_suite * const * const *suites; + + for (suites = suite_set.start; suites < suite_set.end; suites++) + kunit_free_subsuite(*suites); + kfree(suite_set.start); +} + static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, const char *filter_glob) { int i; struct kunit_suite * const **copy, * const *filtered_subsuite; struct suite_set filtered; + struct kunit_test_filter filter; const size_t max = suite_set->end - suite_set->start; @@ -73,12 +155,17 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, return filtered; } + kunit_parse_filter_glob(&filter, filter_glob); + for (i = 0; i < max; ++i) { - filtered_subsuite = kunit_filter_subsuite(suite_set->start[i], filter_glob); + filtered_subsuite = kunit_filter_subsuite(suite_set->start[i], &filter); if (filtered_subsuite) *copy++ = filtered_subsuite; } filtered.end = copy; + + kfree(filter.suite_glob); + kfree(filter.test_glob); return filtered; } @@ -126,9 +213,7 @@ int kunit_run_all_tests(void) __kunit_test_suites_init(*suites); if (filter_glob_param) { /* a copy was made of each array */ - for (suites = suite_set.start; suites < suite_set.end; suites++) - kfree(*suites); - kfree(suite_set.start); + kunit_free_suite_set(suite_set); } kunit_handle_shutdown(); -- cgit From cd94fbc2cafb61cc1394cdba3f92b99ce07b03ae Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Fri, 1 Oct 2021 18:36:35 -0700 Subject: kunit: fix too small allocation when using suite-only kunit.filter_glob When a user filters by a suite and not a test, e.g. $ ./tools/testing/kunit/kunit.py run 'suite_name' it hits this code const int len = strlen(filter_glob); ... parsed->suite_glob = kmalloc(len, GFP_KERNEL); which fails to allocate space for the terminating NULL. Somehow, it seems like we can't easily reproduce this under UML, so the existing `parse_filter_test()` didn't catch this. Fix this by allocating `len + 1` and switch to kzalloc() just to be a bit more defensive. We're only going to run this code once per kernel boot, and it should never be very long. Also update the unit tests to be a bit more cautious. This bug showed up as a NULL pointer dereference here: > KUNIT_EXPECT_STREQ(test, (const char *)filtered.start[0][0]->name, "suite0"); `filtered.start[0][0]` was NULL, and `name` is at offset 0 in the struct, so `...->name` was also NULL. Fixes: 3b29021ddd10 ("kunit: tool: allow filtering test cases via glob") Reported-by: kernel test robot Signed-off-by: Daniel Latypov Reviewed-by: David Gow Acked-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/kunit/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/kunit/executor.c') diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index bab3ab940acc..1d7fecd33261 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -33,7 +33,7 @@ static void kunit_parse_filter_glob(struct kunit_test_filter *parsed, const char *period = strchr(filter_glob, '.'); if (!period) { - parsed->suite_glob = kmalloc(len, GFP_KERNEL); + parsed->suite_glob = kzalloc(len + 1, GFP_KERNEL); parsed->test_glob = NULL; strcpy(parsed->suite_glob, filter_glob); return; -- cgit From 9c6b0e1d8993e47a3fe437af71c6a23d4ee73e12 Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Thu, 30 Sep 2021 15:20:45 -0700 Subject: kunit: add 'kunit.action' param to allow listing out tests Context: It's difficult to map a given .kunitconfig => set of enabled tests. Letting kunit.py figure that out would be useful. This patch: * is intended to be an implementation detail used only by kunit.py * adds a kunit.action module param with one valid non-null value, "list" * for the "list" action, it simply prints out "." * leaves the kunit.py changes to make use of this for another patch. Note: kunit.filter_glob is respected for this and all future actions. Hack: we print a TAP header (but no test plan) to allow kunit.py to use the same code to pick up KUnit output that it does for normal tests. Since this is intended to be an implementation detail, it seems fine for now. Maybe in the future we output each test as SKIPPED or the like. Go with a more generic "action" param, since it seems like we might eventually have more modes besides just running or listing tests, e.g. * perhaps a benchmark mode that reruns test cases and reports timing * perhaps a deflake mode that reruns test cases that failed * perhaps a mode where we randomize test order to try and catch hermeticity bugs like "test a only passes if run after test b" Tested: $ ./tools/testing/kunit/kunit.py run --kernel_arg=kunit.action=list --raw_output=kunit ... TAP version 14 1..1 example.example_simple_test example.example_skip_test example.example_mark_skipped_test reboot: System halted Signed-off-by: Daniel Latypov Reviewed-by: David Gow Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/kunit/executor.c | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) (limited to 'lib/kunit/executor.c') diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index 1d7fecd33261..22640c9ee819 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -15,9 +15,16 @@ extern struct kunit_suite * const * const __kunit_suites_end[]; #if IS_BUILTIN(CONFIG_KUNIT) static char *filter_glob_param; +static char *action_param; + module_param_named(filter_glob, filter_glob_param, charp, 0); MODULE_PARM_DESC(filter_glob, "Filter which KUnit test suites/tests run at boot-time, e.g. list* or list*.*del_test"); +module_param_named(action, action_param, charp, 0); +MODULE_PARM_DESC(action, + "Changes KUnit executor behavior, valid values are:\n" + ": run the tests like normal\n" + "'list' to list test names instead of running them.\n"); /* glob_match() needs NULL terminated strings, so we need a copy of filter_glob_param. */ struct kunit_test_filter { @@ -196,9 +203,35 @@ static void kunit_print_tap_header(struct suite_set *suite_set) pr_info("1..%d\n", num_of_suites); } -int kunit_run_all_tests(void) +static void kunit_exec_run_tests(struct suite_set *suite_set) { struct kunit_suite * const * const *suites; + + kunit_print_tap_header(suite_set); + + for (suites = suite_set->start; suites < suite_set->end; suites++) + __kunit_test_suites_init(*suites); +} + +static void kunit_exec_list_tests(struct suite_set *suite_set) +{ + unsigned int i; + struct kunit_suite * const * const *suites; + struct kunit_case *test_case; + + /* Hack: print a tap header so kunit.py can find the start of KUnit output. */ + pr_info("TAP version 14\n"); + + for (suites = suite_set->start; suites < suite_set->end; suites++) + for (i = 0; (*suites)[i] != NULL; i++) { + kunit_suite_for_each_test_case((*suites)[i], test_case) { + pr_info("%s.%s\n", (*suites)[i]->name, test_case->name); + } + } +} + +int kunit_run_all_tests(void) +{ struct suite_set suite_set = { .start = __kunit_suites_start, .end = __kunit_suites_end, @@ -207,10 +240,12 @@ int kunit_run_all_tests(void) if (filter_glob_param) suite_set = kunit_filter_suites(&suite_set, filter_glob_param); - kunit_print_tap_header(&suite_set); - - for (suites = suite_set.start; suites < suite_set.end; suites++) - __kunit_test_suites_init(*suites); + if (!action_param) + kunit_exec_run_tests(&suite_set); + else if (strcmp(action_param, "list") == 0) + kunit_exec_list_tests(&suite_set); + else + pr_err("kunit executor: unknown action '%s'\n", action_param); if (filter_glob_param) { /* a copy was made of each array */ kunit_free_suite_set(suite_set); -- cgit