From 6ebf5866f2e870cf9451a2068c787bc2b30025e9 Mon Sep 17 00:00:00 2001 From: Felix Guo Date: Mon, 23 Sep 2019 02:02:43 -0700 Subject: kunit: tool: add Python wrappers for running KUnit tests The ultimate goal is to create minimal isolated test binaries; in the meantime we are using UML to provide the infrastructure to run tests, so define an abstract way to configure and run tests that allow us to change the context in which tests are built without affecting the user. This also makes pretty and dynamic error reporting, and a lot of other nice features easier. kunit_config.py: - parse .config and Kconfig files. kunit_kernel.py: provides helper functions to: - configure the kernel using kunitconfig. - build the kernel with the appropriate configuration. - provide function to invoke the kernel and stream the output back. kunit_parser.py: parses raw logs returned out by kunit_kernel and displays them in a user friendly way. test_data/*: samples of test data for testing kunit.py, kunit_config.py, etc. Signed-off-by: Felix Guo Signed-off-by: Brendan Higgins Reviewed-by: Greg Kroah-Hartman Reviewed-by: Logan Gunthorpe Reviewed-by: Stephen Boyd Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit_tool_test.py | 206 +++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100755 tools/testing/kunit/kunit_tool_test.py (limited to 'tools/testing/kunit/kunit_tool_test.py') diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py new file mode 100755 index 000000000000..4a12baa0cd4e --- /dev/null +++ b/tools/testing/kunit/kunit_tool_test.py @@ -0,0 +1,206 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: GPL-2.0 +# +# A collection of tests for tools/testing/kunit/kunit.py +# +# Copyright (C) 2019, Google LLC. +# Author: Brendan Higgins + +import unittest +from unittest import mock + +import tempfile, shutil # Handling test_tmpdir + +import os + +import kunit_config +import kunit_parser +import kunit_kernel +import kunit + +test_tmpdir = '' + +def setUpModule(): + global test_tmpdir + test_tmpdir = tempfile.mkdtemp() + +def tearDownModule(): + shutil.rmtree(test_tmpdir) + +def get_absolute_path(path): + return os.path.join(os.path.dirname(__file__), path) + +class KconfigTest(unittest.TestCase): + + def test_is_subset_of(self): + kconfig0 = kunit_config.Kconfig() + self.assertTrue(kconfig0.is_subset_of(kconfig0)) + + kconfig1 = kunit_config.Kconfig() + kconfig1.add_entry(kunit_config.KconfigEntry('CONFIG_TEST=y')) + self.assertTrue(kconfig1.is_subset_of(kconfig1)) + self.assertTrue(kconfig0.is_subset_of(kconfig1)) + self.assertFalse(kconfig1.is_subset_of(kconfig0)) + + def test_read_from_file(self): + kconfig = kunit_config.Kconfig() + kconfig_path = get_absolute_path( + 'test_data/test_read_from_file.kconfig') + + kconfig.read_from_file(kconfig_path) + + expected_kconfig = kunit_config.Kconfig() + expected_kconfig.add_entry( + kunit_config.KconfigEntry('CONFIG_UML=y')) + expected_kconfig.add_entry( + kunit_config.KconfigEntry('CONFIG_MMU=y')) + expected_kconfig.add_entry( + kunit_config.KconfigEntry('CONFIG_TEST=y')) + expected_kconfig.add_entry( + kunit_config.KconfigEntry('CONFIG_EXAMPLE_TEST=y')) + expected_kconfig.add_entry( + kunit_config.KconfigEntry('# CONFIG_MK8 is not set')) + + self.assertEqual(kconfig.entries(), expected_kconfig.entries()) + + def test_write_to_file(self): + kconfig_path = os.path.join(test_tmpdir, '.config') + + expected_kconfig = kunit_config.Kconfig() + expected_kconfig.add_entry( + kunit_config.KconfigEntry('CONFIG_UML=y')) + expected_kconfig.add_entry( + kunit_config.KconfigEntry('CONFIG_MMU=y')) + expected_kconfig.add_entry( + kunit_config.KconfigEntry('CONFIG_TEST=y')) + expected_kconfig.add_entry( + kunit_config.KconfigEntry('CONFIG_EXAMPLE_TEST=y')) + expected_kconfig.add_entry( + kunit_config.KconfigEntry('# CONFIG_MK8 is not set')) + + expected_kconfig.write_to_file(kconfig_path) + + actual_kconfig = kunit_config.Kconfig() + actual_kconfig.read_from_file(kconfig_path) + + self.assertEqual(actual_kconfig.entries(), + expected_kconfig.entries()) + +class KUnitParserTest(unittest.TestCase): + + def assertContains(self, needle, haystack): + for line in haystack: + if needle in line: + return + raise AssertionError('"' + + str(needle) + '" not found in "' + str(haystack) + '"!') + + def test_output_isolated_correctly(self): + log_path = get_absolute_path( + 'test_data/test_output_isolated_correctly.log') + file = open(log_path) + result = kunit_parser.isolate_kunit_output(file.readlines()) + self.assertContains('TAP version 14\n', result) + self.assertContains(' # Subtest: example', result) + self.assertContains(' 1..2', result) + self.assertContains(' ok 1 - example_simple_test', result) + self.assertContains(' ok 2 - example_mock_test', result) + self.assertContains('ok 1 - example', result) + file.close() + + def test_parse_successful_test_log(self): + all_passed_log = get_absolute_path( + 'test_data/test_is_test_passed-all_passed.log') + file = open(all_passed_log) + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual( + kunit_parser.TestStatus.SUCCESS, + result.status) + file.close() + + def test_parse_failed_test_log(self): + failed_log = get_absolute_path( + 'test_data/test_is_test_passed-failure.log') + file = open(failed_log) + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual( + kunit_parser.TestStatus.FAILURE, + result.status) + file.close() + + def test_no_tests(self): + empty_log = get_absolute_path( + 'test_data/test_is_test_passed-no_tests_run.log') + file = open(empty_log) + result = kunit_parser.parse_run_tests( + kunit_parser.isolate_kunit_output(file.readlines())) + self.assertEqual(0, len(result.suites)) + self.assertEqual( + kunit_parser.TestStatus.NO_TESTS, + result.status) + file.close() + + def test_crashed_test(self): + crashed_log = get_absolute_path( + 'test_data/test_is_test_passed-crash.log') + file = open(crashed_log) + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual( + kunit_parser.TestStatus.TEST_CRASHED, + result.status) + file.close() + +class StrContains(str): + def __eq__(self, other): + return self in other + +class KUnitMainTest(unittest.TestCase): + def setUp(self): + path = get_absolute_path('test_data/test_is_test_passed-all_passed.log') + file = open(path) + all_passed_log = file.readlines() + self.print_patch = mock.patch('builtins.print') + self.print_mock = self.print_patch.start() + self.linux_source_mock = mock.Mock() + self.linux_source_mock.build_reconfig = mock.Mock(return_value=True) + self.linux_source_mock.build_um_kernel = mock.Mock(return_value=True) + self.linux_source_mock.run_kernel = mock.Mock(return_value=all_passed_log) + + def tearDown(self): + self.print_patch.stop() + pass + + def test_run_passes_args_pass(self): + kunit.main(['run'], self.linux_source_mock) + assert self.linux_source_mock.build_reconfig.call_count == 1 + assert self.linux_source_mock.run_kernel.call_count == 1 + self.print_mock.assert_any_call(StrContains('Testing complete.')) + + def test_run_passes_args_fail(self): + self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) + with self.assertRaises(SystemExit) as e: + kunit.main(['run'], self.linux_source_mock) + assert type(e.exception) == SystemExit + assert e.exception.code == 1 + assert self.linux_source_mock.build_reconfig.call_count == 1 + assert self.linux_source_mock.run_kernel.call_count == 1 + self.print_mock.assert_any_call(StrContains(' 0 tests run')) + + def test_run_raw_output(self): + self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) + kunit.main(['run', '--raw_output'], self.linux_source_mock) + assert self.linux_source_mock.build_reconfig.call_count == 1 + assert self.linux_source_mock.run_kernel.call_count == 1 + for kall in self.print_mock.call_args_list: + assert kall != mock.call(StrContains('Testing complete.')) + assert kall != mock.call(StrContains(' 0 tests run')) + + def test_run_timeout(self): + timeout = 3453 + kunit.main(['run', '--timeout', str(timeout)], self.linux_source_mock) + assert self.linux_source_mock.build_reconfig.call_count == 1 + self.linux_source_mock.run_kernel.assert_called_once_with(timeout=timeout) + self.print_mock.assert_any_call(StrContains('Testing complete.')) + +if __name__ == '__main__': + unittest.main() -- cgit From 70efb58bbbc28f0cbd4060c9aa581d79201c8b1a Mon Sep 17 00:00:00 2001 From: Heidi Fahim Date: Mon, 2 Dec 2019 15:53:29 -0800 Subject: kunit: testing kunit: Bug fix in test_run_timeout function Assert in test_run_timeout was not updated with the build_dir argument and caused the following error: AssertionError: Expected call: run_kernel(timeout=3453) Actual call: run_kernel(build_dir=None, timeout=3453) Needed to update kunit_tool_test to reflect this fix https://lkml.org/lkml/2019/9/6/351 Signed-off-by: Heidi Fahim Reviewed-by: SeongJae Park Reviewed-by: Brendan Higgins Tested-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit_tool_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/testing/kunit/kunit_tool_test.py') diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 4a12baa0cd4e..a2a8ea6beae3 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -199,7 +199,7 @@ class KUnitMainTest(unittest.TestCase): timeout = 3453 kunit.main(['run', '--timeout', str(timeout)], self.linux_source_mock) assert self.linux_source_mock.build_reconfig.call_count == 1 - self.linux_source_mock.run_kernel.assert_called_once_with(timeout=timeout) + self.linux_source_mock.run_kernel.assert_called_once_with(build_dir=None, timeout=timeout) self.print_mock.assert_any_call(StrContains('Testing complete.')) if __name__ == '__main__': -- cgit From 609952c2afc9542b336cf9a72d4a09606e05fa2a Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 20 Dec 2019 05:14:06 +0000 Subject: kunit: Place 'test.log' under the 'build_dir' 'kunit' writes the 'test.log' under the kernel source directory even though a 'build_dir' option is given. As users who use the option might expect the outputs to be placed under the specified directory, this commit modifies the logic to write the log file under the 'build_dir'. Signed-off-by: SeongJae Park Reviewed-by: Brendan Higgins Tested-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit_tool_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/testing/kunit/kunit_tool_test.py') diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index a2a8ea6beae3..22f16e66b3c1 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -199,7 +199,7 @@ class KUnitMainTest(unittest.TestCase): timeout = 3453 kunit.main(['run', '--timeout', str(timeout)], self.linux_source_mock) assert self.linux_source_mock.build_reconfig.call_count == 1 - self.linux_source_mock.run_kernel.assert_called_once_with(build_dir=None, timeout=timeout) + self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='', timeout=timeout) self.print_mock.assert_any_call(StrContains('Testing complete.')) if __name__ == '__main__': -- cgit From b1b35201bcf411191986eefdef5f5fb5f41cd879 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 20 Dec 2019 05:14:08 +0000 Subject: kunit/kunit_tool_test: Test '--build_dir' option run This commit adds kunit tool test for the '--build_dir' option. Signed-off-by: SeongJae Park Reviewed-by: Brendan Higgins Tested-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit_tool_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'tools/testing/kunit/kunit_tool_test.py') diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 22f16e66b3c1..cba97756ac4a 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -174,6 +174,7 @@ class KUnitMainTest(unittest.TestCase): kunit.main(['run'], self.linux_source_mock) assert self.linux_source_mock.build_reconfig.call_count == 1 assert self.linux_source_mock.run_kernel.call_count == 1 + self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='', timeout=300) self.print_mock.assert_any_call(StrContains('Testing complete.')) def test_run_passes_args_fail(self): @@ -202,5 +203,12 @@ class KUnitMainTest(unittest.TestCase): self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='', timeout=timeout) self.print_mock.assert_any_call(StrContains('Testing complete.')) + def test_run_builddir(self): + build_dir = '.kunit' + kunit.main(['run', '--build_dir', build_dir], self.linux_source_mock) + assert self.linux_source_mock.build_reconfig.call_count == 1 + self.linux_source_mock.run_kernel.assert_called_once_with(build_dir=build_dir, timeout=300) + self.print_mock.assert_any_call(StrContains('Testing complete.')) + if __name__ == '__main__': unittest.main() -- cgit