diff options
Diffstat (limited to 'tools/testing/selftests/kselftest_harness.h')
| -rw-r--r-- | tools/testing/selftests/kselftest_harness.h | 942 |
1 files changed, 738 insertions, 204 deletions
diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 76d654ef3234..baae6b7ded41 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -1,6 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2012 The Chromium OS Authors. All rights reserved. - * Use of this source code is governed by the GPLv2 license. * * kselftest_harness.h: simple C unit test helper. * @@ -14,7 +14,7 @@ * * .. code-block:: c * - * #include "../kselftest_harness.h" + * #include "kselftest_harness.h" * * TEST(standalone_test) { * do_some_stuff; @@ -50,18 +50,27 @@ #ifndef __KSELFTEST_HARNESS_H #define __KSELFTEST_HARNESS_H +#ifndef _GNU_SOURCE #define _GNU_SOURCE +#endif #include <asm/types.h> +#include <ctype.h> #include <errno.h> +#include <linux/unistd.h> +#include <poll.h> #include <stdbool.h> #include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <sys/mman.h> #include <sys/types.h> #include <sys/wait.h> #include <unistd.h> +#include "kselftest.h" + +#define TEST_TIMEOUT_DEFAULT 30 /* Utilities exposed to the test definitions */ #ifndef TH_LOG_STREAM @@ -73,7 +82,7 @@ #endif /** - * TH_LOG(fmt, ...) + * TH_LOG() * * @fmt: format string * @...: optional arguments @@ -87,14 +96,6 @@ * E.g., #define TH_LOG_ENABLED 1 * * If no definition is provided, logging is enabled by default. - * - * If there is no way to print an error message for the process running the - * test (e.g. not allowed to write to stderr), it is still possible to get the - * ASSERT_* number for which the test failed. This behavior can be enabled by - * writing `_metadata->no_print = true;` before the check sequence that is - * unable to print. When an error occur, instead of printing an error message - * and calling `abort(3)`, the test process call `_exit(2)` with the assert - * number as argument, which is then printed by the parent process. */ #define TH_LOG(fmt, ...) do { \ if (TH_LOG_ENABLED) \ @@ -103,32 +104,37 @@ /* Unconditional logger for internal use. */ #define __TH_LOG(fmt, ...) \ - fprintf(TH_LOG_STREAM, "%s:%d:%s:" fmt "\n", \ + fprintf(TH_LOG_STREAM, "# %s:%d:%s:" fmt "\n", \ __FILE__, __LINE__, _metadata->name, ##__VA_ARGS__) /** - * XFAIL(statement, fmt, ...) + * SKIP() * - * @statement: statement to run after reporting XFAIL + * @statement: statement to run after reporting SKIP * @fmt: format string * @...: optional arguments * - * This forces a "pass" after reporting a failure with an XFAIL prefix, + * .. code-block:: c + * + * SKIP(statement, fmt, ...); + * + * This forces a "pass" after reporting why something is being skipped * and runs "statement", which is usually "return" or "goto skip". */ -#define XFAIL(statement, fmt, ...) do { \ +#define SKIP(statement, fmt, ...) do { \ + snprintf(_metadata->results->reason, \ + sizeof(_metadata->results->reason), fmt, ##__VA_ARGS__); \ if (TH_LOG_ENABLED) { \ - fprintf(TH_LOG_STREAM, "[ XFAIL! ] " fmt "\n", \ - ##__VA_ARGS__); \ + fprintf(TH_LOG_STREAM, "# SKIP %s\n", \ + _metadata->results->reason); \ } \ - /* TODO: find a way to pass xfail to test runner process. */ \ - _metadata->passed = 1; \ + _metadata->exit_code = KSFT_SKIP; \ _metadata->trigger = 0; \ statement; \ } while (0) /** - * TEST(test_name) - Defines the test function and creates the registration + * TEST() - Defines the test function and creates the registration * stub * * @test_name: test name @@ -147,7 +153,7 @@ #define TEST(test_name) __TEST_IMPL(test_name, -1) /** - * TEST_SIGNAL(test_name, signal) + * TEST_SIGNAL() * * @test_name: test name * @signal: signal number @@ -167,9 +173,18 @@ #define __TEST_IMPL(test_name, _signal) \ static void test_name(struct __test_metadata *_metadata); \ + static void wrapper_##test_name( \ + struct __test_metadata *_metadata, \ + struct __fixture_variant_metadata __attribute__((unused)) *variant) \ + { \ + test_name(_metadata); \ + } \ static struct __test_metadata _##test_name##_object = \ - { name: "global." #test_name, \ - fn: &test_name, termsig: _signal }; \ + { .name = #test_name, \ + .fn = &wrapper_##test_name, \ + .fixture = &_fixture_global, \ + .termsig = _signal, \ + .timeout = TEST_TIMEOUT_DEFAULT, }; \ static void __attribute__((constructor)) _register_##test_name(void) \ { \ __register_test(&_##test_name##_object); \ @@ -178,15 +193,16 @@ struct __test_metadata __attribute__((unused)) *_metadata) /** - * FIXTURE_DATA(datatype_name) - Wraps the struct name so we have one less + * FIXTURE_DATA() - Wraps the struct name so we have one less * argument to pass around * * @datatype_name: datatype name * * .. code-block:: c * - * FIXTURE_DATA(datatype name) + * FIXTURE_DATA(datatype_name) * + * Almost always, you want just FIXTURE() instead (see below). * This call may be used when the type of the fixture data * is needed. In general, this should not be needed unless * the *self* is being passed to a helper directly. @@ -194,14 +210,14 @@ #define FIXTURE_DATA(datatype_name) struct _test_data_##datatype_name /** - * FIXTURE(fixture_name) - Called once per fixture to setup the data and + * FIXTURE() - Called once per fixture to setup the data and * register * * @fixture_name: fixture name * * .. code-block:: c * - * FIXTURE(datatype name) { + * FIXTURE(fixture_name) { * type property1; * ... * }; @@ -210,22 +226,25 @@ * populated and cleaned up using FIXTURE_SETUP() and FIXTURE_TEARDOWN(). */ #define FIXTURE(fixture_name) \ + FIXTURE_VARIANT(fixture_name); \ + static struct __fixture_metadata _##fixture_name##_fixture_object = \ + { .name = #fixture_name, }; \ static void __attribute__((constructor)) \ _register_##fixture_name##_data(void) \ { \ - __fixture_count++; \ + __register_fixture(&_##fixture_name##_fixture_object); \ } \ FIXTURE_DATA(fixture_name) /** - * FIXTURE_SETUP(fixture_name) - Prepares the setup function for the fixture. - * *_metadata* is included so that EXPECT_* and ASSERT_* work correctly. + * FIXTURE_SETUP() - Prepares the setup function for the fixture. + * *_metadata* is included so that EXPECT_*, ASSERT_* etc. work correctly. * * @fixture_name: fixture name * * .. code-block:: c * - * FIXTURE_SETUP(fixture name) { implementation } + * FIXTURE_SETUP(fixture_name) { implementation } * * Populates the required "setup" function for a fixture. An instance of the * datatype defined with FIXTURE_DATA() will be exposed as *self* for the @@ -237,18 +256,21 @@ * A bare "return;" statement may be used to return early. */ #define FIXTURE_SETUP(fixture_name) \ - void fixture_name##_setup( \ + static void fixture_name##_setup( \ struct __test_metadata __attribute__((unused)) *_metadata, \ - FIXTURE_DATA(fixture_name) __attribute__((unused)) *self) + FIXTURE_DATA(fixture_name) __attribute__((unused)) *self, \ + const FIXTURE_VARIANT(fixture_name) \ + __attribute__((unused)) *variant) + /** - * FIXTURE_TEARDOWN(fixture_name) - * *_metadata* is included so that EXPECT_* and ASSERT_* work correctly. + * FIXTURE_TEARDOWN() + * *_metadata* is included so that EXPECT_*, ASSERT_* etc. work correctly. * * @fixture_name: fixture name * * .. code-block:: c * - * FIXTURE_TEARDOWN(fixture name) { implementation } + * FIXTURE_TEARDOWN(fixture_name) { implementation } * * Populates the required "teardown" function for a fixture. An instance of the * datatype defined with FIXTURE_DATA() will be exposed as *self* for the @@ -257,12 +279,93 @@ * A bare "return;" statement may be used to return early. */ #define FIXTURE_TEARDOWN(fixture_name) \ - void fixture_name##_teardown( \ + static const bool fixture_name##_teardown_parent; \ + __FIXTURE_TEARDOWN(fixture_name) + +/** + * FIXTURE_TEARDOWN_PARENT() + * *_metadata* is included so that EXPECT_*, ASSERT_* etc. work correctly. + * + * @fixture_name: fixture name + * + * .. code-block:: c + * + * FIXTURE_TEARDOWN_PARENT(fixture_name) { implementation } + * + * Same as FIXTURE_TEARDOWN() but run this code in a parent process. This + * enables the test process to drop its privileges without impacting the + * related FIXTURE_TEARDOWN_PARENT() (e.g. to remove files from a directory + * where write access was dropped). + * + * To make it possible for the parent process to use *self*, share (MAP_SHARED) + * the fixture data between all forked processes. + */ +#define FIXTURE_TEARDOWN_PARENT(fixture_name) \ + static const bool fixture_name##_teardown_parent = true; \ + __FIXTURE_TEARDOWN(fixture_name) + +#define __FIXTURE_TEARDOWN(fixture_name) \ + static void fixture_name##_teardown( \ struct __test_metadata __attribute__((unused)) *_metadata, \ - FIXTURE_DATA(fixture_name) __attribute__((unused)) *self) + FIXTURE_DATA(fixture_name) __attribute__((unused)) *self, \ + const FIXTURE_VARIANT(fixture_name) \ + __attribute__((unused)) *variant) + +/** + * FIXTURE_VARIANT() - Optionally called once per fixture + * to declare fixture variant + * + * @fixture_name: fixture name + * + * .. code-block:: c + * + * FIXTURE_VARIANT(fixture_name) { + * type property1; + * ... + * }; + * + * Defines type of constant parameters provided to FIXTURE_SETUP(), TEST_F() and + * FIXTURE_TEARDOWN as *variant*. Variants allow the same tests to be run with + * different arguments. + */ +#define FIXTURE_VARIANT(fixture_name) struct _fixture_variant_##fixture_name + +/** + * FIXTURE_VARIANT_ADD() - Called once per fixture + * variant to setup and register the data + * + * @fixture_name: fixture name + * @variant_name: name of the parameter set + * + * .. code-block:: c + * + * FIXTURE_VARIANT_ADD(fixture_name, variant_name) { + * .property1 = val1, + * ... + * }; + * + * Defines a variant of the test fixture, provided to FIXTURE_SETUP() and + * TEST_F() as *variant*. Tests of each fixture will be run once for each + * variant. + */ +#define FIXTURE_VARIANT_ADD(fixture_name, variant_name) \ + extern const FIXTURE_VARIANT(fixture_name) \ + _##fixture_name##_##variant_name##_variant; \ + static struct __fixture_variant_metadata \ + _##fixture_name##_##variant_name##_object = \ + { .name = #variant_name, \ + .data = &_##fixture_name##_##variant_name##_variant}; \ + static void __attribute__((constructor)) \ + _register_##fixture_name##_##variant_name(void) \ + { \ + __register_fixture_variant(&_##fixture_name##_fixture_object, \ + &_##fixture_name##_##variant_name##_object); \ + } \ + const FIXTURE_VARIANT(fixture_name) \ + _##fixture_name##_##variant_name##_variant = /** - * TEST_F(fixture_name, test_name) - Emits test registration and helpers for + * TEST_F() - Emits test registration and helpers for * fixture-based test cases * * @fixture_name: fixture name @@ -276,46 +379,105 @@ * Very similar to TEST() except that *self* is the setup instance of fixture's * datatype exposed for use by the implementation. * - * Warning: use of ASSERT_* here will skip TEARDOWN. + * The _metadata object is shared (MAP_SHARED) with all the potential forked + * processes, which enables them to use EXCEPT_*() and ASSERT_*(). + * + * The *self* object is only shared with the potential forked processes if + * FIXTURE_TEARDOWN_PARENT() is used instead of FIXTURE_TEARDOWN(). */ -/* TODO(wad) register fixtures on dedicated test lists. */ #define TEST_F(fixture_name, test_name) \ - __TEST_F_IMPL(fixture_name, test_name, -1) + __TEST_F_IMPL(fixture_name, test_name, -1, TEST_TIMEOUT_DEFAULT) #define TEST_F_SIGNAL(fixture_name, test_name, signal) \ - __TEST_F_IMPL(fixture_name, test_name, signal) + __TEST_F_IMPL(fixture_name, test_name, signal, TEST_TIMEOUT_DEFAULT) -#define __TEST_F_IMPL(fixture_name, test_name, signal) \ +#define TEST_F_TIMEOUT(fixture_name, test_name, timeout) \ + __TEST_F_IMPL(fixture_name, test_name, -1, timeout) + +#define __TEST_F_IMPL(fixture_name, test_name, signal, tmout) \ static void fixture_name##_##test_name( \ struct __test_metadata *_metadata, \ - FIXTURE_DATA(fixture_name) *self); \ - static inline void wrapper_##fixture_name##_##test_name( \ - struct __test_metadata *_metadata) \ + FIXTURE_DATA(fixture_name) *self, \ + const FIXTURE_VARIANT(fixture_name) *variant); \ + static void wrapper_##fixture_name##_##test_name( \ + struct __test_metadata *_metadata, \ + struct __fixture_variant_metadata *variant) \ { \ /* fixture data is alloced, setup, and torn down per call. */ \ - FIXTURE_DATA(fixture_name) self; \ - memset(&self, 0, sizeof(FIXTURE_DATA(fixture_name))); \ - fixture_name##_setup(_metadata, &self); \ - /* Let setup failure terminate early. */ \ - if (!_metadata->passed) \ - return; \ - fixture_name##_##test_name(_metadata, &self); \ - fixture_name##_teardown(_metadata, &self); \ + FIXTURE_DATA(fixture_name) self_private, *self = NULL; \ + pid_t child = 1; \ + int status = 0; \ + /* Makes sure there is only one teardown, even when child forks again. */ \ + _metadata->no_teardown = mmap(NULL, sizeof(*_metadata->no_teardown), \ + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); \ + *_metadata->no_teardown = true; \ + if (sizeof(*self) > 0) { \ + if (fixture_name##_teardown_parent) { \ + self = mmap(NULL, sizeof(*self), PROT_READ | PROT_WRITE, \ + MAP_SHARED | MAP_ANONYMOUS, -1, 0); \ + } else { \ + memset(&self_private, 0, sizeof(self_private)); \ + self = &self_private; \ + } \ + } \ + _metadata->variant = variant->data; \ + _metadata->self = self; \ + /* _metadata and potentially self are shared with all forks. */ \ + child = fork(); \ + if (child == 0) { \ + fixture_name##_setup(_metadata, self, variant->data); \ + /* Let setup failure terminate early. */ \ + if (_metadata->exit_code) \ + _exit(0); \ + *_metadata->no_teardown = false; \ + fixture_name##_##test_name(_metadata, self, variant->data); \ + _metadata->teardown_fn(false, _metadata, self, variant->data); \ + _exit(0); \ + } else if (child < 0 || child != waitpid(child, &status, 0)) { \ + ksft_print_msg("ERROR SPAWNING TEST GRANDCHILD\n"); \ + _metadata->exit_code = KSFT_FAIL; \ + } \ + _metadata->teardown_fn(true, _metadata, self, variant->data); \ + munmap(_metadata->no_teardown, sizeof(*_metadata->no_teardown)); \ + _metadata->no_teardown = NULL; \ + if (self && fixture_name##_teardown_parent) \ + munmap(self, sizeof(*self)); \ + if (WIFEXITED(status)) { \ + if (WEXITSTATUS(status)) \ + _metadata->exit_code = WEXITSTATUS(status); \ + } else if (WIFSIGNALED(status)) { \ + /* Forward signal to __wait_for_test(). */ \ + kill(getpid(), WTERMSIG(status)); \ + } \ } \ - static struct __test_metadata \ - _##fixture_name##_##test_name##_object = { \ - name: #fixture_name "." #test_name, \ - fn: &wrapper_##fixture_name##_##test_name, \ - termsig: signal, \ - }; \ + static void wrapper_##fixture_name##_##test_name##_teardown( \ + bool in_parent, struct __test_metadata *_metadata, \ + void *self, const void *variant) \ + { \ + if (fixture_name##_teardown_parent == in_parent && \ + !__atomic_test_and_set(_metadata->no_teardown, __ATOMIC_RELAXED)) \ + fixture_name##_teardown(_metadata, self, variant); \ + } \ + static struct __test_metadata *_##fixture_name##_##test_name##_object; \ static void __attribute__((constructor)) \ _register_##fixture_name##_##test_name(void) \ { \ - __register_test(&_##fixture_name##_##test_name##_object); \ + struct __test_metadata *object = mmap(NULL, sizeof(*object), \ + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); \ + object->name = #test_name; \ + object->fn = &wrapper_##fixture_name##_##test_name; \ + object->fixture = &_##fixture_name##_fixture_object; \ + object->teardown_fn = &wrapper_##fixture_name##_##test_name##_teardown; \ + object->termsig = signal; \ + object->timeout = tmout; \ + _##fixture_name##_##test_name##_object = object; \ + __register_test(object); \ } \ static void fixture_name##_##test_name( \ struct __test_metadata __attribute__((unused)) *_metadata, \ - FIXTURE_DATA(fixture_name) __attribute__((unused)) *self) + FIXTURE_DATA(fixture_name) __attribute__((unused)) *self, \ + const FIXTURE_VARIANT(fixture_name) \ + __attribute__((unused)) *variant) /** * TEST_HARNESS_MAIN - Simple wrapper to run the test harness @@ -327,12 +489,6 @@ * Use once to append a main() to the test file. */ #define TEST_HARNESS_MAIN \ - static void __attribute__((constructor)) \ - __constructor_order_last(void) \ - { \ - if (!__constructor_order) \ - __constructor_order = _CONSTRUCTOR_ORDER_BACKWARD; \ - } \ int main(int argc, char **argv) { \ return test_harness_run(argc, argv); \ } @@ -346,7 +502,7 @@ */ /** - * ASSERT_EQ(expected, seen) + * ASSERT_EQ() * * @expected: expected value * @seen: measured value @@ -357,7 +513,7 @@ __EXPECT(expected, #expected, seen, #seen, ==, 1) /** - * ASSERT_NE(expected, seen) + * ASSERT_NE() * * @expected: expected value * @seen: measured value @@ -368,7 +524,7 @@ __EXPECT(expected, #expected, seen, #seen, !=, 1) /** - * ASSERT_LT(expected, seen) + * ASSERT_LT() * * @expected: expected value * @seen: measured value @@ -379,7 +535,7 @@ __EXPECT(expected, #expected, seen, #seen, <, 1) /** - * ASSERT_LE(expected, seen) + * ASSERT_LE() * * @expected: expected value * @seen: measured value @@ -390,7 +546,7 @@ __EXPECT(expected, #expected, seen, #seen, <=, 1) /** - * ASSERT_GT(expected, seen) + * ASSERT_GT() * * @expected: expected value * @seen: measured value @@ -401,7 +557,7 @@ __EXPECT(expected, #expected, seen, #seen, >, 1) /** - * ASSERT_GE(expected, seen) + * ASSERT_GE() * * @expected: expected value * @seen: measured value @@ -412,7 +568,7 @@ __EXPECT(expected, #expected, seen, #seen, >=, 1) /** - * ASSERT_NULL(seen) + * ASSERT_NULL() * * @seen: measured value * @@ -422,7 +578,7 @@ __EXPECT(NULL, "NULL", seen, #seen, ==, 1) /** - * ASSERT_TRUE(seen) + * ASSERT_TRUE() * * @seen: measured value * @@ -432,7 +588,7 @@ __EXPECT(0, "0", seen, #seen, !=, 1) /** - * ASSERT_FALSE(seen) + * ASSERT_FALSE() * * @seen: measured value * @@ -442,7 +598,7 @@ __EXPECT(0, "0", seen, #seen, ==, 1) /** - * ASSERT_STREQ(expected, seen) + * ASSERT_STREQ() * * @expected: expected value * @seen: measured value @@ -453,7 +609,7 @@ __EXPECT_STR(expected, seen, ==, 1) /** - * ASSERT_STRNE(expected, seen) + * ASSERT_STRNE() * * @expected: expected value * @seen: measured value @@ -464,7 +620,7 @@ __EXPECT_STR(expected, seen, !=, 1) /** - * EXPECT_EQ(expected, seen) + * EXPECT_EQ() * * @expected: expected value * @seen: measured value @@ -475,7 +631,7 @@ __EXPECT(expected, #expected, seen, #seen, ==, 0) /** - * EXPECT_NE(expected, seen) + * EXPECT_NE() * * @expected: expected value * @seen: measured value @@ -486,7 +642,7 @@ __EXPECT(expected, #expected, seen, #seen, !=, 0) /** - * EXPECT_LT(expected, seen) + * EXPECT_LT() * * @expected: expected value * @seen: measured value @@ -497,7 +653,7 @@ __EXPECT(expected, #expected, seen, #seen, <, 0) /** - * EXPECT_LE(expected, seen) + * EXPECT_LE() * * @expected: expected value * @seen: measured value @@ -508,7 +664,7 @@ __EXPECT(expected, #expected, seen, #seen, <=, 0) /** - * EXPECT_GT(expected, seen) + * EXPECT_GT() * * @expected: expected value * @seen: measured value @@ -519,7 +675,7 @@ __EXPECT(expected, #expected, seen, #seen, >, 0) /** - * EXPECT_GE(expected, seen) + * EXPECT_GE() * * @expected: expected value * @seen: measured value @@ -530,7 +686,7 @@ __EXPECT(expected, #expected, seen, #seen, >=, 0) /** - * EXPECT_NULL(seen) + * EXPECT_NULL() * * @seen: measured value * @@ -540,7 +696,7 @@ __EXPECT(NULL, "NULL", seen, #seen, ==, 0) /** - * EXPECT_TRUE(seen) + * EXPECT_TRUE() * * @seen: measured value * @@ -550,7 +706,7 @@ __EXPECT(0, "0", seen, #seen, !=, 0) /** - * EXPECT_FALSE(seen) + * EXPECT_FALSE() * * @seen: measured value * @@ -560,7 +716,7 @@ __EXPECT(0, "0", seen, #seen, ==, 0) /** - * EXPECT_STREQ(expected, seen) + * EXPECT_STREQ() * * @expected: expected value * @seen: measured value @@ -571,7 +727,7 @@ __EXPECT_STR(expected, seen, ==, 0) /** - * EXPECT_STRNE(expected, seen) + * EXPECT_STRNE() * * @expected: expected value * @seen: measured value @@ -581,7 +737,9 @@ #define EXPECT_STRNE(expected, seen) \ __EXPECT_STR(expected, seen, !=, 0) +#ifndef ARRAY_SIZE #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) +#endif /* Support an optional handler after and ASSERT_* or EXPECT_*. The approach is * not thread-safe, but it should be fine in most sane test scenarios. @@ -591,24 +749,51 @@ */ #define OPTIONAL_HANDLER(_assert) \ for (; _metadata->trigger; _metadata->trigger = \ - __bail(_assert, _metadata->no_print, _metadata->step)) + __bail(_assert, _metadata)) -#define __INC_STEP(_metadata) \ - if (_metadata->passed && _metadata->step < 255) \ - _metadata->step++; +#define is_signed_var(var) (!!(((__typeof__(var))(-1)) < (__typeof__(var))1)) #define __EXPECT(_expected, _expected_str, _seen, _seen_str, _t, _assert) do { \ /* Avoid multiple evaluation of the cases */ \ __typeof__(_expected) __exp = (_expected); \ __typeof__(_seen) __seen = (_seen); \ - if (_assert) __INC_STEP(_metadata); \ if (!(__exp _t __seen)) { \ - unsigned long long __exp_print = (uintptr_t)__exp; \ - unsigned long long __seen_print = (uintptr_t)__seen; \ - __TH_LOG("Expected %s (%llu) %s %s (%llu)", \ - _expected_str, __exp_print, #_t, \ - _seen_str, __seen_print); \ - _metadata->passed = 0; \ + /* Report with actual signedness to avoid weird output. */ \ + switch (is_signed_var(__exp) * 2 + is_signed_var(__seen)) { \ + case 0: { \ + uintmax_t __exp_print = (uintmax_t)__exp; \ + uintmax_t __seen_print = (uintmax_t)__seen; \ + __TH_LOG("Expected %s (%ju) %s %s (%ju)", \ + _expected_str, __exp_print, #_t, \ + _seen_str, __seen_print); \ + break; \ + } \ + case 1: { \ + uintmax_t __exp_print = (uintmax_t)__exp; \ + intmax_t __seen_print = (intmax_t)__seen; \ + __TH_LOG("Expected %s (%ju) %s %s (%jd)", \ + _expected_str, __exp_print, #_t, \ + _seen_str, __seen_print); \ + break; \ + } \ + case 2: { \ + intmax_t __exp_print = (intmax_t)__exp; \ + uintmax_t __seen_print = (uintmax_t)__seen; \ + __TH_LOG("Expected %s (%jd) %s %s (%ju)", \ + _expected_str, __exp_print, #_t, \ + _seen_str, __seen_print); \ + break; \ + } \ + case 3: { \ + intmax_t __exp_print = (intmax_t)__exp; \ + intmax_t __seen_print = (intmax_t)__seen; \ + __TH_LOG("Expected %s (%jd) %s %s (%jd)", \ + _expected_str, __exp_print, #_t, \ + _seen_str, __seen_print); \ + break; \ + } \ + } \ + _metadata->exit_code = KSFT_FAIL; \ /* Ensure the optional handler is triggered */ \ _metadata->trigger = 1; \ } \ @@ -617,163 +802,512 @@ #define __EXPECT_STR(_expected, _seen, _t, _assert) do { \ const char *__exp = (_expected); \ const char *__seen = (_seen); \ - if (_assert) __INC_STEP(_metadata); \ if (!(strcmp(__exp, __seen) _t 0)) { \ __TH_LOG("Expected '%s' %s '%s'.", __exp, #_t, __seen); \ - _metadata->passed = 0; \ + _metadata->exit_code = KSFT_FAIL; \ _metadata->trigger = 1; \ } \ } while (0); OPTIONAL_HANDLER(_assert) +/* List helpers */ +#define __LIST_APPEND(head, item) \ +{ \ + /* Circular linked list where only prev is circular. */ \ + if (head == NULL) { \ + head = item; \ + item->next = NULL; \ + item->prev = item; \ + return; \ + } \ + if (__constructor_order_forward) { \ + item->next = NULL; \ + item->prev = head->prev; \ + item->prev->next = item; \ + head->prev = item; \ + } else { \ + item->next = head; \ + item->next->prev = item; \ + item->prev = item; \ + head = item; \ + } \ +} + +struct __test_results { + char reason[1024]; /* Reason for test result */ +}; + +struct __test_metadata; +struct __fixture_variant_metadata; + +/* Contains all the information about a fixture. */ +struct __fixture_metadata { + const char *name; + struct __test_metadata *tests; + struct __fixture_variant_metadata *variant; + struct __fixture_metadata *prev, *next; +} _fixture_global __attribute__((unused)) = { + .name = "global", + .prev = &_fixture_global, +}; + +struct __test_xfail { + struct __fixture_metadata *fixture; + struct __fixture_variant_metadata *variant; + struct __test_metadata *test; + struct __test_xfail *prev, *next; +}; + +/** + * XFAIL_ADD() - mark variant + test case combination as expected to fail + * @fixture_name: name of the fixture + * @variant_name: name of the variant + * @test_name: name of the test case + * + * Mark a combination of variant + test case for a given fixture as expected + * to fail. Tests marked this way will report XPASS / XFAIL return codes, + * instead of PASS / FAIL,and use respective counters. + */ +#define XFAIL_ADD(fixture_name, variant_name, test_name) \ + static struct __test_xfail \ + _##fixture_name##_##variant_name##_##test_name##_xfail = \ + { \ + .fixture = &_##fixture_name##_fixture_object, \ + .variant = &_##fixture_name##_##variant_name##_object, \ + }; \ + static void __attribute__((constructor)) \ + _register_##fixture_name##_##variant_name##_##test_name##_xfail(void) \ + { \ + _##fixture_name##_##variant_name##_##test_name##_xfail.test = \ + _##fixture_name##_##test_name##_object; \ + __register_xfail(&_##fixture_name##_##variant_name##_##test_name##_xfail); \ + } + +static struct __fixture_metadata *__fixture_list = &_fixture_global; +static bool __constructor_order_forward; + +static inline void __register_fixture(struct __fixture_metadata *f) +{ + __LIST_APPEND(__fixture_list, f); +} + +struct __fixture_variant_metadata { + const char *name; + const void *data; + struct __test_xfail *xfails; + struct __fixture_variant_metadata *prev, *next; +}; + +static inline void +__register_fixture_variant(struct __fixture_metadata *f, + struct __fixture_variant_metadata *variant) +{ + __LIST_APPEND(f->variant, variant); +} + /* Contains all the information for test execution and status checking. */ struct __test_metadata { const char *name; - void (*fn)(struct __test_metadata *); + void (*fn)(struct __test_metadata *, + struct __fixture_variant_metadata *); + pid_t pid; /* pid of test when being run */ + struct __fixture_metadata *fixture; + void (*teardown_fn)(bool in_parent, struct __test_metadata *_metadata, + void *self, const void *variant); int termsig; - int passed; + int exit_code; int trigger; /* extra handler after the evaluation */ - __u8 step; - bool no_print; /* manual trigger when TH_LOG_STREAM is not available */ + int timeout; /* seconds to wait for test timeout */ + bool aborted; /* stopped test due to failed ASSERT */ + bool *no_teardown; /* fixture needs teardown */ + void *self; + const void *variant; + struct __test_results *results; struct __test_metadata *prev, *next; }; -/* Storage for the (global) tests to be run. */ -static struct __test_metadata *__test_list; -static unsigned int __test_count; -static unsigned int __fixture_count; -static int __constructor_order; - -#define _CONSTRUCTOR_ORDER_FORWARD 1 -#define _CONSTRUCTOR_ORDER_BACKWARD -1 +static inline bool __test_passed(struct __test_metadata *metadata) +{ + return metadata->exit_code != KSFT_FAIL && + metadata->exit_code <= KSFT_SKIP; +} /* * Since constructors are called in reverse order, reverse the test * list so tests are run in source declaration order. * https://gcc.gnu.org/onlinedocs/gccint/Initialization.html * However, it seems not all toolchains do this correctly, so use - * __constructor_order to detect which direction is called first + * __constructor_order_foward to detect which direction is called first * and adjust list building logic to get things running in the right * direction. */ static inline void __register_test(struct __test_metadata *t) { - __test_count++; - /* Circular linked list where only prev is circular. */ - if (__test_list == NULL) { - __test_list = t; - t->next = NULL; - t->prev = t; - return; - } - if (__constructor_order == _CONSTRUCTOR_ORDER_FORWARD) { - t->next = NULL; - t->prev = __test_list->prev; - t->prev->next = t; - __test_list->prev = t; - } else { - t->next = __test_list; - t->next->prev = t; - t->prev = t; - __test_list = t; - } + __LIST_APPEND(t->fixture->tests, t); +} + +static inline void __register_xfail(struct __test_xfail *xf) +{ + __LIST_APPEND(xf->variant->xfails, xf); } -static inline int __bail(int for_realz, bool no_print, __u8 step) +static inline int __bail(int for_realz, struct __test_metadata *t) { + /* if this is ASSERT, return immediately. */ if (for_realz) { - if (no_print) - _exit(step); + if (t->teardown_fn) + t->teardown_fn(false, t, t->self, t->variant); abort(); } + /* otherwise, end the for loop and continue. */ return 0; } -void __run_test(struct __test_metadata *t) +static void __wait_for_test(struct __test_metadata *t) { - pid_t child_pid; - int status; + /* + * Sets status so that WIFEXITED(status) returns true and + * WEXITSTATUS(status) returns KSFT_FAIL. This safe default value + * should never be evaluated because of the waitpid(2) check and + * timeout handling. + */ + int status = KSFT_FAIL << 8; + struct pollfd poll_child; + int ret, child, childfd; + bool timed_out = false; - t->passed = 1; - t->trigger = 0; - printf("[ RUN ] %s\n", t->name); - child_pid = fork(); - if (child_pid < 0) { - printf("ERROR SPAWNING TEST CHILD\n"); - t->passed = 0; - } else if (child_pid == 0) { - t->fn(t); - /* return the step that failed or 0 */ - _exit(t->passed ? 0 : t->step); - } else { - /* TODO(wad) add timeout support. */ - waitpid(child_pid, &status, 0); - if (WIFEXITED(status)) { - t->passed = t->termsig == -1 ? !WEXITSTATUS(status) : 0; - if (t->termsig != -1) { - fprintf(TH_LOG_STREAM, - "%s: Test exited normally " - "instead of by signal (code: %d)\n", - t->name, - WEXITSTATUS(status)); - } else if (!t->passed) { - fprintf(TH_LOG_STREAM, - "%s: Test failed at step #%d\n", - t->name, - WEXITSTATUS(status)); - } - } else if (WIFSIGNALED(status)) { - t->passed = 0; - if (WTERMSIG(status) == SIGABRT) { + childfd = syscall(__NR_pidfd_open, t->pid, 0); + if (childfd == -1) { + t->exit_code = KSFT_FAIL; + fprintf(TH_LOG_STREAM, + "# %s: unable to open pidfd\n", + t->name); + return; + } + + poll_child.fd = childfd; + poll_child.events = POLLIN; + ret = poll(&poll_child, 1, t->timeout * 1000); + if (ret == -1) { + t->exit_code = KSFT_FAIL; + fprintf(TH_LOG_STREAM, + "# %s: unable to wait on child pidfd\n", + t->name); + return; + } else if (ret == 0) { + timed_out = true; + /* signal process group */ + kill(-(t->pid), SIGKILL); + } + child = waitpid(t->pid, &status, WNOHANG); + if (child == -1 && errno != EINTR) { + t->exit_code = KSFT_FAIL; + fprintf(TH_LOG_STREAM, + "# %s: Failed to wait for PID %d (errno: %d)\n", + t->name, t->pid, errno); + return; + } + + if (timed_out) { + t->exit_code = KSFT_FAIL; + fprintf(TH_LOG_STREAM, + "# %s: Test terminated by timeout\n", t->name); + } else if (WIFEXITED(status)) { + if (WEXITSTATUS(status) == KSFT_SKIP || + WEXITSTATUS(status) == KSFT_XPASS || + WEXITSTATUS(status) == KSFT_XFAIL) { + t->exit_code = WEXITSTATUS(status); + } else if (t->termsig != -1) { + t->exit_code = KSFT_FAIL; + fprintf(TH_LOG_STREAM, + "# %s: Test exited normally instead of by signal (code: %d)\n", + t->name, + WEXITSTATUS(status)); + } else { + switch (WEXITSTATUS(status)) { + /* Success */ + case KSFT_PASS: + t->exit_code = KSFT_PASS; + break; + /* Failure */ + default: + t->exit_code = KSFT_FAIL; fprintf(TH_LOG_STREAM, - "%s: Test terminated by assertion\n", + "# %s: Test failed\n", t->name); - } else if (WTERMSIG(status) == t->termsig) { - t->passed = 1; - } else { - fprintf(TH_LOG_STREAM, - "%s: Test terminated unexpectedly " - "by signal %d\n", - t->name, - WTERMSIG(status)); } + } + } else if (WIFSIGNALED(status)) { + t->exit_code = KSFT_FAIL; + if (WTERMSIG(status) == SIGABRT) { + fprintf(TH_LOG_STREAM, + "# %s: Test terminated by assertion\n", + t->name); + } else if (WTERMSIG(status) == t->termsig) { + t->exit_code = KSFT_PASS; } else { fprintf(TH_LOG_STREAM, - "%s: Test ended in some other way [%u]\n", + "# %s: Test terminated unexpectedly by signal %d\n", t->name, - status); + WTERMSIG(status)); } + } else { + t->exit_code = KSFT_FAIL; + fprintf(TH_LOG_STREAM, + "# %s: Test ended in some other way [%u]\n", + t->name, + status); } - printf("[ %4s ] %s\n", (t->passed ? "OK" : "FAIL"), t->name); } -static int test_harness_run(int __attribute__((unused)) argc, - char __attribute__((unused)) **argv) +static void test_harness_list_tests(void) { + struct __fixture_variant_metadata *v; + struct __fixture_metadata *f; struct __test_metadata *t; - int ret = 0; + + for (f = __fixture_list; f; f = f->next) { + v = f->variant; + t = f->tests; + + if (f == __fixture_list) + fprintf(stderr, "%-20s %-25s %s\n", + "# FIXTURE", "VARIANT", "TEST"); + else + fprintf(stderr, "--------------------------------------------------------------------------------\n"); + + do { + fprintf(stderr, "%-20s %-25s %s\n", + t == f->tests ? f->name : "", + v ? v->name : "", + t ? t->name : ""); + + v = v ? v->next : NULL; + t = t ? t->next : NULL; + } while (v || t); + } +} + +static int test_harness_argv_check(int argc, char **argv) +{ + int opt; + + while ((opt = getopt(argc, argv, "dhlF:f:V:v:t:T:r:")) != -1) { + switch (opt) { + case 'f': + case 'F': + case 'v': + case 'V': + case 't': + case 'T': + case 'r': + break; + case 'l': + test_harness_list_tests(); + return KSFT_SKIP; + case 'd': + ksft_debug_enabled = true; + break; + case 'h': + default: + fprintf(stderr, + "Usage: %s [-h|-l|-d] [-t|-T|-v|-V|-f|-F|-r name]\n" + "\t-h print help\n" + "\t-l list all tests\n" + "\t-d enable debug prints\n" + "\n" + "\t-t name include test\n" + "\t-T name exclude test\n" + "\t-v name include variant\n" + "\t-V name exclude variant\n" + "\t-f name include fixture\n" + "\t-F name exclude fixture\n" + "\t-r name run specified test\n" + "\n" + "Test filter options can be specified " + "multiple times. The filtering stops\n" + "at the first match. For example to " + "include all tests from variant 'bla'\n" + "but not test 'foo' specify '-T foo -v bla'.\n" + "", argv[0]); + return opt == 'h' ? KSFT_SKIP : KSFT_FAIL; + } + } + + return KSFT_PASS; +} + +static bool test_enabled(int argc, char **argv, + struct __fixture_metadata *f, + struct __fixture_variant_metadata *v, + struct __test_metadata *t) +{ + unsigned int flen = 0, vlen = 0, tlen = 0; + bool has_positive = false; + int opt; + + optind = 1; + while ((opt = getopt(argc, argv, "dF:f:V:v:t:T:r:")) != -1) { + if (opt != 'd') + has_positive |= islower(opt); + + switch (tolower(opt)) { + case 't': + if (!strcmp(t->name, optarg)) + return islower(opt); + break; + case 'f': + if (!strcmp(f->name, optarg)) + return islower(opt); + break; + case 'v': + if (!strcmp(v->name, optarg)) + return islower(opt); + break; + case 'r': + if (!tlen) { + flen = strlen(f->name); + vlen = strlen(v->name); + tlen = strlen(t->name); + } + if (strlen(optarg) == flen + 1 + vlen + !!vlen + tlen && + !strncmp(f->name, &optarg[0], flen) && + !strncmp(v->name, &optarg[flen + 1], vlen) && + !strncmp(t->name, &optarg[flen + 1 + vlen + !!vlen], tlen)) + return true; + break; + } + } + + /* + * If there are no positive tests then we assume user just wants + * exclusions and everything else is a pass. + */ + return !has_positive; +} + +static void __run_test(struct __fixture_metadata *f, + struct __fixture_variant_metadata *variant, + struct __test_metadata *t) +{ + struct __test_xfail *xfail; + char test_name[1024]; + const char *diagnostic; + int child; + + /* reset test struct */ + t->exit_code = KSFT_PASS; + t->trigger = 0; + t->aborted = false; + t->no_teardown = NULL; + memset(t->results->reason, 0, sizeof(t->results->reason)); + + snprintf(test_name, sizeof(test_name), "%s%s%s.%s", + f->name, variant->name[0] ? "." : "", variant->name, t->name); + + ksft_print_msg(" RUN %s ...\n", test_name); + + /* Make sure output buffers are flushed before fork */ + fflush(stdout); + fflush(stderr); + + child = fork(); + if (child < 0) { + ksft_print_msg("ERROR SPAWNING TEST CHILD\n"); + t->exit_code = KSFT_FAIL; + } else if (child == 0) { + setpgrp(); + t->fn(t, variant); + _exit(t->exit_code); + } else { + t->pid = child; + __wait_for_test(t); + } + ksft_print_msg(" %4s %s\n", + __test_passed(t) ? "OK" : "FAIL", test_name); + + /* Check if we're expecting this test to fail */ + for (xfail = variant->xfails; xfail; xfail = xfail->next) + if (xfail->test == t) + break; + if (xfail) + t->exit_code = __test_passed(t) ? KSFT_XPASS : KSFT_XFAIL; + + if (t->results->reason[0]) + diagnostic = t->results->reason; + else if (t->exit_code == KSFT_PASS || t->exit_code == KSFT_FAIL) + diagnostic = NULL; + else + diagnostic = "unknown"; + + ksft_test_result_code(t->exit_code, test_name, + diagnostic ? "%s" : NULL, diagnostic); +} + +static int test_harness_run(int argc, char **argv) +{ + struct __fixture_variant_metadata no_variant = { .name = "", }; + struct __fixture_variant_metadata *v; + struct __fixture_metadata *f; + struct __test_results *results; + struct __test_metadata *t; + int ret; + unsigned int case_count = 0, test_count = 0; unsigned int count = 0; unsigned int pass_count = 0; - /* TODO(wad) add optional arguments similar to gtest. */ - printf("[==========] Running %u tests from %u test cases.\n", - __test_count, __fixture_count + 1); - for (t = __test_list; t; t = t->next) { - count++; - __run_test(t); - if (t->passed) - pass_count++; - else - ret = 1; + ret = test_harness_argv_check(argc, argv); + if (ret != KSFT_PASS) + return ret; + + for (f = __fixture_list; f; f = f->next) { + for (v = f->variant ?: &no_variant; v; v = v->next) { + unsigned int old_tests = test_count; + + for (t = f->tests; t; t = t->next) + if (test_enabled(argc, argv, f, v, t)) + test_count++; + + if (old_tests != test_count) + case_count++; + } + } + + results = mmap(NULL, sizeof(*results), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + + ksft_print_header(); + ksft_set_plan(test_count); + ksft_print_msg("Starting %u tests from %u test cases.\n", + test_count, case_count); + for (f = __fixture_list; f; f = f->next) { + for (v = f->variant ?: &no_variant; v; v = v->next) { + for (t = f->tests; t; t = t->next) { + if (!test_enabled(argc, argv, f, v, t)) + continue; + count++; + t->results = results; + __run_test(f, v, t); + t->results = NULL; + if (__test_passed(t)) + pass_count++; + else + ret = 1; + } + } } - printf("[==========] %u / %u tests passed.\n", pass_count, count); - printf("[ %s ]\n", (ret ? "FAILED" : "PASSED")); - return ret; + munmap(results, sizeof(*results)); + + ksft_print_msg("%s: %u / %u tests passed.\n", ret ? "FAILED" : "PASSED", + pass_count, count); + ksft_exit(ret == 0); + + /* unreachable */ + return KSFT_FAIL; } static void __attribute__((constructor)) __constructor_order_first(void) { - if (!__constructor_order) - __constructor_order = _CONSTRUCTOR_ORDER_FORWARD; + __constructor_order_forward = true; } #endif /* __KSELFTEST_HARNESS_H */ |