- Linux-kselftest-mirror - lists.linaro.org

[PATCH 1/2] kunit: tool: move Kconfig read_from_file/parse_from_string to package-level

by Daniel Latypov

read_from_file() clears its `self` Kconfig object and parses a config file. It is a way to construct Kconfig objects more so than an operation on Kconfig objects. This is reflected in the fact its only ever used as: kconfig = kunit_config.Kconfig() kconfig.read_from_file(path) So clean this up and simplify callers by replacing it with kconfig = kunit_config.parse_file(path) Do the same thing for the related parse_from_string() function as well. Signed-off-by: Daniel Latypov <dlatypov(a)google.com> --- tools/testing/kunit/kunit_config.py | 61 +++++++++++++------------- tools/testing/kunit/kunit_kernel.py | 12 ++--- tools/testing/kunit/kunit_tool_test.py | 6 +-- 3 files changed, 37 insertions(+), 42 deletions(-) diff --git a/tools/testing/kunit/kunit_config.py b/tools/testing/kunit/kunit_config.py index c77c7d2ef622..677354546156 100644 --- a/tools/testing/kunit/kunit_config.py +++ b/tools/testing/kunit/kunit_config.py @@ -62,33 +62,34 @@ class Kconfig(object): for entry in self.entries(): f.write(str(entry) + '\n') - def parse_from_string(self, blob: str) -> None: - """Parses a string containing KconfigEntrys and populates this Kconfig.""" - self._entries = [] - is_not_set_matcher = re.compile(CONFIG_IS_NOT_SET_PATTERN) - config_matcher = re.compile(CONFIG_PATTERN) - for line in blob.split('\n'): - line = line.strip() - if not line: - continue - - match = config_matcher.match(line) - if match: - entry = KconfigEntry(match.group(1), match.group(2)) - self.add_entry(entry) - continue - - empty_match = is_not_set_matcher.match(line) - if empty_match: - entry = KconfigEntry(empty_match.group(1), 'n') - self.add_entry(entry) - continue - - if line[0] == '#': - continue - else: - raise KconfigParseError('Failed to parse: ' + line) - - def read_from_file(self, path: str) -> None: - with open(path, 'r') as f: - self.parse_from_string(f.read()) +def parse_file(path: str) -> Kconfig: + with open(path, 'r') as f: + return parse_from_string(f.read()) + +def parse_from_string(blob: str) -> Kconfig: + """Parses a string containing Kconfig entries.""" + kconfig = Kconfig() + is_not_set_matcher = re.compile(CONFIG_IS_NOT_SET_PATTERN) + config_matcher = re.compile(CONFIG_PATTERN) + for line in blob.split('\n'): + line = line.strip() + if not line: + continue + + match = config_matcher.match(line) + if match: + entry = KconfigEntry(match.group(1), match.group(2)) + kconfig.add_entry(entry) + continue + + empty_match = is_not_set_matcher.match(line) + if empty_match: + entry = KconfigEntry(empty_match.group(1), 'n') + kconfig.add_entry(entry) + continue + + if line[0] == '#': + continue + else: + raise KconfigParseError('Failed to parse: ' + line) + return kconfig diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index 66095568bf32..51ee6e5dae91 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -116,8 +116,7 @@ class LinuxSourceTreeOperationsQemu(LinuxSourceTreeOperations): self._extra_qemu_params = qemu_arch_params.extra_qemu_params def make_arch_qemuconfig(self, base_kunitconfig: kunit_config.Kconfig) -> None: - kconfig = kunit_config.Kconfig() - kconfig.parse_from_string(self._kconfig) + kconfig = kunit_config.parse_from_string(self._kconfig) base_kunitconfig.merge_in_entries(kconfig) def start(self, params: List[str], build_dir: str) -> subprocess.Popen: @@ -249,8 +248,7 @@ class LinuxSourceTree(object): if not os.path.exists(kunitconfig_path): shutil.copyfile(DEFAULT_KUNITCONFIG_PATH, kunitconfig_path) - self._kconfig = kunit_config.Kconfig() - self._kconfig.read_from_file(kunitconfig_path) + self._kconfig = kunit_config.parse_file(kunitconfig_path) def clean(self) -> bool: try: @@ -262,8 +260,7 @@ class LinuxSourceTree(object): def validate_config(self, build_dir) -> bool: kconfig_path = get_kconfig_path(build_dir) - validated_kconfig = kunit_config.Kconfig() - validated_kconfig.read_from_file(kconfig_path) + validated_kconfig = kunit_config.parse_file(kconfig_path) if not self._kconfig.is_subset_of(validated_kconfig): invalid = self._kconfig.entries() - validated_kconfig.entries() message = 'Provided Kconfig is not contained in validated .config. Following fields found in kunitconfig, ' \ @@ -291,8 +288,7 @@ class LinuxSourceTree(object): """Creates a new .config if it is not a subset of the .kunitconfig.""" kconfig_path = get_kconfig_path(build_dir) if os.path.exists(kconfig_path): - existing_kconfig = kunit_config.Kconfig() - existing_kconfig.read_from_file(kconfig_path) + existing_kconfig = kunit_config.parse_file(kconfig_path) self._ops.make_arch_qemuconfig(self._kconfig) if not self._kconfig.is_subset_of(existing_kconfig): print('Regenerating .config ...') diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 9c4126731457..4ec70e41ec5a 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -50,10 +50,9 @@ class KconfigTest(unittest.TestCase): self.assertFalse(kconfig1.is_subset_of(kconfig0)) def test_read_from_file(self): - kconfig = kunit_config.Kconfig() kconfig_path = test_data_path('test_read_from_file.kconfig') - kconfig.read_from_file(kconfig_path) + kconfig = kunit_config.parse_file(kconfig_path) expected_kconfig = kunit_config.Kconfig() expected_kconfig.add_entry( @@ -86,8 +85,7 @@ class KconfigTest(unittest.TestCase): expected_kconfig.write_to_file(kconfig_path) - actual_kconfig = kunit_config.Kconfig() - actual_kconfig.read_from_file(kconfig_path) + actual_kconfig = kunit_config.parse_file(kconfig_path) self.assertEqual(actual_kconfig.entries(), expected_kconfig.entries()) base-commit: 52a5d80a2225e2d0b2a8f4656b76aead2a443b2a -- 2.34.0.rc0.344.g81b53c2807-goog

4 years

3
8
0 0

[PATCH 1/2] kunit: tool: use dataclass instead of collections.namedtuple

by Daniel Latypov

namedtuple is a terse way of defining a collection of fields. However, it does not allow us to annotate the type of these fields. It also doesn't let us have any sort of inheritance between types. Since commit df4b0807ca1a ("kunit: tool: Assert the version requirement"), kunit.py has asserted that it's running on python >=3.7. So in that case use a 3.7 feature, dataclasses, to replace these. Changes in detail: * Make KunitExecRequest contain all the fields needed for exec_tests * Use inheritance to dedupe fields * also allows us to e.g. pass a KUnitRequest in as a KUnitParseRequest * this has changed around the order of some fields * Use named arguments when constructing all request objects in kunit.py * This is to prevent accidentally mixing up fields, etc. Signed-off-by: Daniel Latypov <dlatypov(a)google.com> --- tools/testing/kunit/kunit.py | 139 +++++++++++++------------ tools/testing/kunit/kunit_tool_test.py | 6 +- 2 files changed, 75 insertions(+), 70 deletions(-) diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 9c9ed4071e9e..f879414a13c4 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -15,38 +15,57 @@ import time assert sys.version_info >= (3, 7), "Python version is too old" -from collections import namedtuple +from dataclasses import dataclass from enum import Enum, auto -from typing import Iterable, List +from typing import Any, Iterable, List, Optional import kunit_json import kunit_kernel import kunit_parser -KunitResult = namedtuple('KunitResult', ['status','result','elapsed_time']) - -KunitConfigRequest = namedtuple('KunitConfigRequest', - ['build_dir', 'make_options']) -KunitBuildRequest = namedtuple('KunitBuildRequest', - ['jobs', 'build_dir', 'alltests', - 'make_options']) -KunitExecRequest = namedtuple('KunitExecRequest', - ['timeout', 'build_dir', 'alltests', - 'filter_glob', 'kernel_args', 'run_isolated']) -KunitParseRequest = namedtuple('KunitParseRequest', - ['raw_output', 'build_dir', 'json']) -KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', - 'build_dir', 'alltests', 'filter_glob', - 'kernel_args', 'run_isolated', 'json', 'make_options']) - -KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] - class KunitStatus(Enum): SUCCESS = auto() CONFIG_FAILURE = auto() BUILD_FAILURE = auto() TEST_FAILURE = auto() +@dataclass +class KunitResult: + status: KunitStatus + result: Any + elapsed_time: float + +@dataclass +class KunitConfigRequest: + build_dir: str + make_options: Optional[List[str]] + +@dataclass +class KunitBuildRequest(KunitConfigRequest): + jobs: int + alltests: bool + +@dataclass +class KunitParseRequest: + raw_output: Optional[str] + build_dir: str + json: Optional[str] + +@dataclass +class KunitExecRequest(KunitParseRequest): + timeout: int + alltests: bool + filter_glob: str + kernel_args: Optional[List[str]] + run_isolated: Optional[str] + +@dataclass +class KunitRequest(KunitExecRequest, KunitBuildRequest): + pass + + +KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] + def get_kernel_root_path() -> str: path = sys.argv[0] if not __file__ else __file__ parts = os.path.realpath(path).split('tools/testing/kunit') @@ -121,8 +140,7 @@ def _suites_from_test_list(tests: List[str]) -> List[str]: -def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest, - parse_request: KunitParseRequest) -> KunitResult: +def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult: filter_globs = [request.filter_glob] if request.run_isolated: tests = _list_tests(linux, request) @@ -147,7 +165,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest, filter_glob=filter_glob, build_dir=request.build_dir) - result = parse_tests(parse_request, run_result) + result = parse_tests(request, run_result) # run_kernel() doesn't block on the kernel exiting. # That only happens after we get the last line of output from `run_result`. # So exec_time here actually contains parsing + execution time, which is fine. @@ -211,27 +229,15 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitRequest) -> KunitResult: run_start = time.time() - config_request = KunitConfigRequest(request.build_dir, - request.make_options) - config_result = config_tests(linux, config_request) + config_result = config_tests(linux, request) if config_result.status != KunitStatus.SUCCESS: return config_result - build_request = KunitBuildRequest(request.jobs, request.build_dir, - request.alltests, - request.make_options) - build_result = build_tests(linux, build_request) + build_result = build_tests(linux, request) if build_result.status != KunitStatus.SUCCESS: return build_result - exec_request = KunitExecRequest(request.timeout, request.build_dir, - request.alltests, request.filter_glob, - request.kernel_args, request.run_isolated) - parse_request = KunitParseRequest(request.raw_output, - request.build_dir, - request.json) - - exec_result = exec_tests(linux, exec_request, parse_request) + exec_result = exec_tests(linux, request) run_end = time.time() @@ -382,16 +388,16 @@ def main(argv, linux=None): cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) - request = KunitRequest(cli_args.raw_output, - cli_args.timeout, - cli_args.jobs, - cli_args.build_dir, - cli_args.alltests, - cli_args.filter_glob, - cli_args.kernel_args, - cli_args.run_isolated, - cli_args.json, - cli_args.make_options) + request = KunitRequest(build_dir=cli_args.build_dir, + make_options=cli_args.make_options, + jobs=cli_args.jobs, + alltests=cli_args.alltests, + raw_output=cli_args.raw_output, + json=cli_args.json, + timeout=cli_args.timeout, + filter_glob=cli_args.filter_glob, + kernel_args=cli_args.kernel_args, + run_isolated=cli_args.run_isolated) result = run_tests(linux, request) if result.status != KunitStatus.SUCCESS: sys.exit(1) @@ -407,8 +413,8 @@ def main(argv, linux=None): cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) - request = KunitConfigRequest(cli_args.build_dir, - cli_args.make_options) + request = KunitConfigRequest(build_dir=cli_args.build_dir, + make_options=cli_args.make_options) result = config_tests(linux, request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % ( @@ -423,10 +429,10 @@ def main(argv, linux=None): cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) - request = KunitBuildRequest(cli_args.jobs, - cli_args.build_dir, - cli_args.alltests, - cli_args.make_options) + request = KunitBuildRequest(build_dir=cli_args.build_dir, + make_options=cli_args.make_options, + jobs=cli_args.jobs, + alltests=cli_args.alltests) result = build_tests(linux, request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % ( @@ -441,16 +447,15 @@ def main(argv, linux=None): cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) - exec_request = KunitExecRequest(cli_args.timeout, - cli_args.build_dir, - cli_args.alltests, - cli_args.filter_glob, - cli_args.kernel_args, - cli_args.run_isolated) - parse_request = KunitParseRequest(cli_args.raw_output, - cli_args.build_dir, - cli_args.json) - result = exec_tests(linux, exec_request, parse_request) + exec_request = KunitExecRequest(raw_output=cli_args.raw_output, + build_dir=cli_args.build_dir, + json=cli_args.json, + timeout=cli_args.timeout, + alltests=cli_args.alltests, + filter_glob=cli_args.filter_glob, + kernel_args=cli_args.kernel_args, + run_isolated=cli_args.run_isolated) + result = exec_tests(linux, exec_request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % (result.elapsed_time)) if result.status != KunitStatus.SUCCESS: @@ -461,9 +466,9 @@ def main(argv, linux=None): else: with open(cli_args.file, 'r') as f: kunit_output = f.read().splitlines() - request = KunitParseRequest(cli_args.raw_output, - None, - cli_args.json) + request = KunitParseRequest(raw_output=cli_args.raw_output, + build_dir='', + json=cli_args.json) result = parse_tests(request, kunit_output) if result.status != KunitStatus.SUCCESS: sys.exit(1) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 6648de1f9ceb..2540bb10b4e8 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -556,7 +556,7 @@ class KUnitMainTest(unittest.TestCase): self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want got = kunit._list_tests(self.linux_source_mock, - kunit.KunitExecRequest(300, '.kunit', False, 'suite*', None, 'suite')) + kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*', None, 'suite')) self.assertEqual(got, want) # Should respect the user's filter glob when listing tests. @@ -571,7 +571,7 @@ class KUnitMainTest(unittest.TestCase): # Should respect the user's filter glob when listing tests. mock_tests.assert_called_once_with(mock.ANY, - kunit.KunitExecRequest(300, '.kunit', False, 'suite*.test*', None, 'suite')) + kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*.test*', None, 'suite')) self.linux_source_mock.run_kernel.assert_has_calls([ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', timeout=300), @@ -584,7 +584,7 @@ class KUnitMainTest(unittest.TestCase): # Should respect the user's filter glob when listing tests. mock_tests.assert_called_once_with(mock.ANY, - kunit.KunitExecRequest(300, '.kunit', False, 'suite*', None, 'test')) + kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*', None, 'test')) self.linux_source_mock.run_kernel.assert_has_calls([ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', timeout=300), base-commit: e3c6457b588d83b7ecd40eb4bd6d95007020fbe4 -- 2.33.0.882.g93a45727a2-goog

4 years

2
4
0 0

[PATCH v2] kunit: tool: print parsed test results fully incrementally

by Daniel Latypov

With the parser rework [1] and run_kernel() rework [2], this allows the parser to print out test results incrementally. Currently, that's held up by the fact that the LineStream eagerly pre-fetches the next line when you call pop(). This blocks parse_test_result() from returning until the line *after* the "ok 1 - test name" line is also printed. One can see this with the following example: $ (echo -e 'TAP version 14\n1..3\nok 1 - fake test'; sleep 2; echo -e 'ok 2 - fake test 2'; sleep 3; echo -e 'ok 3 - fake test 3') | ./tools/testing/kunit/kunit.py parse Before this patch [1]: there's a pause before 'fake test' is printed. After this patch: 'fake test' is printed out immediately. This patch also adds * a unit test to verify LineStream's behavior directly * a test case to ensure that it's lazily calling the generator * an explicit exception for when users go beyond EOF [1] https://lore.kernel.org/linux-kselftest/20211006170049.106852-1-dlatypov@go… [2] https://lore.kernel.org/linux-kselftest/20211005011340.2826268-1-dlatypov@g… Signed-off-by: Daniel Latypov <dlatypov(a)google.com> Reviewed-by: David Gow <davidgow(a)google.com> --- v1 -> v2: rebase onto v7 of parser rewrite https://lore.kernel.org/all/20211007210324.707912-1-dlatypov@google.com/ --- tools/testing/kunit/kunit_parser.py | 22 ++++++++++---- tools/testing/kunit/kunit_tool_test.py | 42 +++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index f01fd565f978..82900a5f9ad6 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -172,42 +172,51 @@ class TestCounts: class LineStream: """ A class to represent the lines of kernel output. - Provides a peek()/pop() interface over an iterator of + Provides a lazy peek()/pop() interface over an iterator of (line#, text). """ _lines: Iterator[Tuple[int, str]] _next: Tuple[int, str] + _need_next: bool _done: bool def __init__(self, lines: Iterator[Tuple[int, str]]): """Creates a new LineStream that wraps the given iterator.""" self._lines = lines self._done = False + self._need_next = True self._next = (0, '') - self._get_next() def _get_next(self) -> None: - """Advances the LineSteam to the next line.""" + """Advances the LineSteam to the next line, if necessary.""" + if not self._need_next: + return try: self._next = next(self._lines) except StopIteration: self._done = True + finally: + self._need_next = False def peek(self) -> str: """Returns the current line, without advancing the LineStream. """ + self._get_next() return self._next[1] def pop(self) -> str: """Returns the current line and advances the LineStream to the next line. """ - n = self._next - self._get_next() - return n[1] + s = self.peek() + if self._done: + raise ValueError(f'LineStream: going past EOF, last line was {s}') + self._need_next = True + return s def __bool__(self) -> bool: """Returns True if stream has more lines.""" + self._get_next() return not self._done # Only used by kunit_tool_test.py. @@ -220,6 +229,7 @@ class LineStream: def line_number(self) -> int: """Returns the line number of the current line.""" + self._get_next() return self._next[0] # Parsing helper methods: diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 6648de1f9ceb..77e61b0a40e8 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -13,9 +13,10 @@ import tempfile, shutil # Handling test_tmpdir import itertools import json +import os import signal import subprocess -import os +from typing import Iterable import kunit_config import kunit_parser @@ -319,6 +320,45 @@ class KUnitParserTest(unittest.TestCase): result.status) self.assertEqual('kunit-resource-test', result.test.subtests[0].name) +def line_stream_from_strs(strs: Iterable[str]) -> kunit_parser.LineStream: + return kunit_parser.LineStream(enumerate(strs, start=1)) + +class LineStreamTest(unittest.TestCase): + + def test_basic(self): + stream = line_stream_from_strs(['hello', 'world']) + + self.assertTrue(stream, msg='Should be more input') + self.assertEqual(stream.line_number(), 1) + self.assertEqual(stream.peek(), 'hello') + self.assertEqual(stream.pop(), 'hello') + + self.assertTrue(stream, msg='Should be more input') + self.assertEqual(stream.line_number(), 2) + self.assertEqual(stream.peek(), 'world') + self.assertEqual(stream.pop(), 'world') + + self.assertFalse(stream, msg='Should be no more input') + with self.assertRaisesRegex(ValueError, 'LineStream: going past EOF'): + stream.pop() + + def test_is_lazy(self): + called_times = 0 + def generator(): + nonlocal called_times + for i in range(1,5): + called_times += 1 + yield called_times, str(called_times) + + stream = kunit_parser.LineStream(generator()) + self.assertEqual(called_times, 0) + + self.assertEqual(stream.pop(), '1') + self.assertEqual(called_times, 1) + + self.assertEqual(stream.pop(), '2') + self.assertEqual(called_times, 2) + class LinuxSourceTreeTest(unittest.TestCase): def setUp(self): base-commit: e3c6457b588d83b7ecd40eb4bd6d95007020fbe4 -- 2.33.0.882.g93a45727a2-goog

4 years

2
1
0 0

[PATCH v4 1/4] kunit: tool: Do not error on tests without test plans

by David Gow

The (K)TAP spec encourages test output to begin with a 'test plan': a count of the number of tests being run of the form: 1..n However, some test suites might not know the number of subtests in advance (for example, KUnit's parameterised tests use a generator function). In this case, it's not possible to print the test plan in advance. kunit_tool already parses test output which doesn't contain a plan, but reports an error. Since we want to use nested subtests with KUnit paramterised tests, remove this error. Signed-off-by: David Gow <davidgow(a)google.com> Reviewed-by: Daniel Latypov <dlatypov(a)google.com> --- No changes since v3: https://lore.kernel.org/linux-kselftest/20211028064154.2301049-1-davidgow@g… Changes since v2: https://lore.kernel.org/linux-kselftest/20211027013702.2039566-1-davidgow@g… - No code changes. - Added Daniel's Reviewed-by. tools/testing/kunit/kunit_parser.py | 5 ++--- tools/testing/kunit/kunit_tool_test.py | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 3355196d0515..50ded55c168c 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -340,8 +340,8 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: """ Parses test plan line and stores the expected number of subtests in test object. Reports an error if expected count is 0. - Returns False and reports missing test plan error if fails to parse - test plan. + Returns False and sets expected_count to None if there is no valid test + plan. Accepted format: - '1..[number of subtests]' @@ -356,7 +356,6 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: match = TEST_PLAN.match(lines.peek()) if not match: test.expected_count = None - test.add_error('missing plan line!') return False test.log.append(lines.pop()) expected_count = int(match.group(1)) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 9c4126731457..bc8793145713 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -191,7 +191,10 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines( file.readlines())) - self.assertEqual(2, result.test.counts.errors) + # A missing test plan is not an error. + self.assertEqual(0, result.test.counts.errors) + # All tests should be accounted for. + self.assertEqual(10, result.test.counts.total()) self.assertEqual( kunit_parser.TestStatus.SUCCESS, result.status) -- 2.33.1.1089.g2158813163f-goog

4 years

2
7
0 0

[PATCH v2] kunit: add run_checks.py script to validate kunit changes

by Daniel Latypov

This formalizes the checks KUnit maintainers have been running (or in other cases: forgetting to run). This script also runs them all in parallel to minimize friction (pytype can be fairly slow, but not slower than running kunit.py). Example output: $ ./tools/testing/kunit/run_checks.py Waiting on 4 checks (kunit_tool_test.py, kunit smoke test, pytype, mypy)... kunit_tool_test.py: PASSED mypy: PASSED pytype: PASSED kunit smoke test: PASSED On failure or timeout (5 minutes), it'll dump out the stdout/stderr. E.g. adding in a type-checking error: mypy: FAILED > kunit.py:54: error: Name 'nonexistent_function' is not defined > Found 1 error in 1 file (checked 8 source files) mypy and pytype are two Python type-checkers and must be installed. This file treats them as optional and will mark them as SKIPPED if not installed. This tool also runs `kunit.py run --kunitconfig=lib/kunit` to run KUnit's own KUnit tests and to verify KUnit kernel code and kunit.py play nicely together. It uses --build_dir=kunit_run_checks so as not to clobber the default build_dir, which helps make it faster by reducing the need to rebuild, esp. if you're been passing in --arch instead of using UML. Signed-off-by: Daniel Latypov <dlatypov(a)google.com> Reviewed-by: David Gow <davidgow(a)google.com> --- tools/testing/kunit/run_checks.py | 81 +++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 tools/testing/kunit/run_checks.py diff --git a/tools/testing/kunit/run_checks.py b/tools/testing/kunit/run_checks.py new file mode 100755 index 000000000000..4f32133ed77c --- /dev/null +++ b/tools/testing/kunit/run_checks.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# +# This file runs some basic checks to verify kunit works. +# It is only of interest if you're making changes to KUnit itself. +# +# Copyright (C) 2021, Google LLC. +# Author: Daniel Latypov <dlatypov(a)google.com.com> + +from concurrent import futures +import datetime +import os +import shutil +import subprocess +import sys +import textwrap +from typing import Dict, List, Sequence, Tuple + +ABS_TOOL_PATH = os.path.abspath(os.path.dirname(__file__)) +TIMEOUT = datetime.timedelta(minutes=5).total_seconds() + +commands: Dict[str, Sequence[str]] = { + 'kunit_tool_test.py': ['./kunit_tool_test.py'], + 'kunit smoke test': ['./kunit.py', 'run', '--kunitconfig=lib/kunit', '--build_dir=kunit_run_checks'], + 'pytype': ['/bin/sh', '-c', 'pytype *.py'], + 'mypy': ['/bin/sh', '-c', 'mypy *.py'], +} + +# The user might not have mypy or pytype installed, skip them if so. +# Note: you can install both via `$ pip install mypy pytype` +necessary_deps : Dict[str, str] = { + 'pytype': 'pytype', + 'mypy': 'mypy', +} + +def main(argv: Sequence[str]) -> None: + if argv: + raise RuntimeError('This script takes no arguments') + + future_to_name: Dict[futures.Future, str] = {} + executor = futures.ThreadPoolExecutor(max_workers=len(commands)) + for name, argv in commands.items(): + if name in necessary_deps and shutil.which(necessary_deps[name]) is None: + print(f'{name}: SKIPPED, {necessary_deps[name]} not in $PATH') + continue + f = executor.submit(run_cmd, argv) + future_to_name[f] = name + + has_failures = False + print(f'Waiting on {len(future_to_name)} checks ({", ".join(future_to_name.values())})...') + for f in futures.as_completed(future_to_name.keys()): + name = future_to_name[f] + ex = f.exception() + if not ex: + print(f'{name}: PASSED') + continue + + has_failures = True + if isinstance(ex, subprocess.TimeoutExpired): + print(f'{name}: TIMED OUT') + elif isinstance(ex, subprocess.CalledProcessError): + print(f'{name}: FAILED') + else: + print('{name}: unexpected exception: {ex}') + continue + + output = ex.output + if output: + print(textwrap.indent(output.decode(), '> ')) + executor.shutdown() + + if has_failures: + sys.exit(1) + + +def run_cmd(argv: Sequence[str]): + subprocess.check_output(argv, stderr=subprocess.STDOUT, cwd=ABS_TOOL_PATH, timeout=TIMEOUT) + + +if __name__ == '__main__': + main(sys.argv[1:]) base-commit: 52a5d80a2225e2d0b2a8f4656b76aead2a443b2a -- 2.33.1.1089.g2158813163f-goog

4 years

3
2
0 0

[PATCH] Documentation: kunit: remove claims that kunit is a mocking framework

by Daniel Latypov

KUnit does not have any first party support for "mocking". The original RFC had some, but the code got dropped. However, the documentation patches never got updated. This fixes that. https://kunit.dev/mocking.html has a current writeup on the status quo and will hopefully be eventually folded into the in-kernel Documentation. Signed-off-by: Daniel Latypov <dlatypov(a)google.com> --- Documentation/dev-tools/kunit/api/index.rst | 3 +-- Documentation/dev-tools/kunit/api/test.rst | 3 +-- Documentation/dev-tools/kunit/index.rst | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Documentation/dev-tools/kunit/api/index.rst b/Documentation/dev-tools/kunit/api/index.rst index b33ad72bcf0b..3006cadcf44a 100644 --- a/Documentation/dev-tools/kunit/api/index.rst +++ b/Documentation/dev-tools/kunit/api/index.rst @@ -12,5 +12,4 @@ following sections: Documentation/dev-tools/kunit/api/test.rst - - documents all of the standard testing API excluding mocking - or mocking related features. + - documents all of the standard testing API diff --git a/Documentation/dev-tools/kunit/api/test.rst b/Documentation/dev-tools/kunit/api/test.rst index aaa97f17e5b3..c5eca423e8b6 100644 --- a/Documentation/dev-tools/kunit/api/test.rst +++ b/Documentation/dev-tools/kunit/api/test.rst @@ -4,8 +4,7 @@ Test API ======== -This file documents all of the standard testing API excluding mocking or mocking -related features. +This file documents all of the standard testing API. .. kernel-doc:: include/kunit/test.h :internal: diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst index cacb35ec658d..7af7dec83646 100644 --- a/Documentation/dev-tools/kunit/index.rst +++ b/Documentation/dev-tools/kunit/index.rst @@ -19,7 +19,7 @@ KUnit - Unit Testing for the Linux Kernel What is KUnit? ============== -KUnit is a lightweight unit testing and mocking framework for the Linux kernel. +KUnit is a lightweight unit testing framework for the Linux kernel. KUnit is heavily inspired by JUnit, Python's unittest.mock, and Googletest/Googlemock for C++. KUnit provides facilities for defining unit test base-commit: 2ab5d5e67f7ab2d2ecf67b8855ac65691f4e4b4d -- 2.33.0.1079.g6e70778dc9-goog

4 years

3
2
0 0

[PATCH] kunit: tool: fix --json output for skipped tests

by Daniel Latypov

Currently, KUnit will report SKIPPED tests as having failed if one uses --json. Add the missing if statement to set the appropriate status ("SKIP"). See https://api.kernelci.org/schema-test-case.html: "status": { "type": "string", "description": "The status of the execution of this test case", "enum": ["PASS", "FAIL", "SKIP", "ERROR"], "default": "PASS" }, with this, we now can properly produce all four of the statuses. Fixes: 5acaf6031f53 ("kunit: tool: Support skipped tests in kunit_tool") Signed-off-by: Daniel Latypov <dlatypov(a)google.com> --- tools/testing/kunit/kunit_json.py | 2 ++ tools/testing/kunit/kunit_tool_test.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/tools/testing/kunit/kunit_json.py b/tools/testing/kunit/kunit_json.py index 746bec72b9ac..b6e66c5d64d1 100644 --- a/tools/testing/kunit/kunit_json.py +++ b/tools/testing/kunit/kunit_json.py @@ -30,6 +30,8 @@ def _get_group_json(test: Test, def_config: str, test_case = {"name": subtest.name, "status": "FAIL"} if subtest.status == TestStatus.SUCCESS: test_case["status"] = "PASS" + elif subtest.status == TestStatus.SKIPPED: + test_case["status"] = "SKIP" elif subtest.status == TestStatus.TEST_CRASHED: test_case["status"] = "ERROR" test_cases.append(test_case) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 77e61b0a40e8..b3cc0227843c 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -426,6 +426,12 @@ class KUnitJsonTest(unittest.TestCase): {'name': 'example_simple_test', 'status': 'ERROR'}, result["sub_groups"][1]["test_cases"][0]) + def test_skipped_test_json(self): + result = self._json_for('test_skip_tests.log') + self.assertEqual( + {'name': 'example_skip_test', 'status': 'SKIP'}, + result["sub_groups"][1]["test_cases"][1]) + def test_no_tests_json(self): result = self._json_for('test_is_test_passed-no_tests_run_with_header.log') self.assertEqual(0, len(result['sub_groups'])) base-commit: 4a86e2973c31902a2a72f4f25f99b2367188ec5d -- 2.33.0.882.g93a45727a2-goog

4 years

3
2
0 0

[PATCH v1 0/7] Documentation: KUnit: Rework KUnit documentation

by Harinder Singh

The KUnit documentation was not very organized. There was little information related to KUnit architecture and the importance of unit testing. Add some new pages, expand and reorganize the existing documentation. Reword pages to make information and style more consistent. Harinder Singh (7): Documentation: KUnit: Rewrite main page Documentation: KUnit: Rewrite getting started Documentation: KUnit: Added KUnit Architecture Documentation: kunit: Reorganize documentation related to running tests Documentation: KUnit: Rework writing page to focus on writing tests Documentation: KUnit: Restyle Test Style and Nomenclature page Documentation: KUnit: Restyled Frequently Asked Questions .../dev-tools/kunit/architecture.rst | 206 +++++++ Documentation/dev-tools/kunit/faq.rst | 73 ++- Documentation/dev-tools/kunit/index.rst | 165 ++--- .../kunit/kunit_suitememorydiagram.png | Bin 0 -> 24174 bytes Documentation/dev-tools/kunit/run_manual.rst | 57 ++ Documentation/dev-tools/kunit/run_wrapper.rst | 247 ++++++++ Documentation/dev-tools/kunit/start.rst | 195 +++--- Documentation/dev-tools/kunit/style.rst | 101 ++-- Documentation/dev-tools/kunit/usage.rst | 570 ++++++++---------- 9 files changed, 1028 insertions(+), 586 deletions(-) create mode 100644 Documentation/dev-tools/kunit/architecture.rst create mode 100644 Documentation/dev-tools/kunit/kunit_suitememorydiagram.png create mode 100644 Documentation/dev-tools/kunit/run_manual.rst create mode 100644 Documentation/dev-tools/kunit/run_wrapper.rst base-commit: 4c388a8e740d3235a194f330c8ef327deef710f6 -- 2.34.0.384.gca35af8252-goog

4 years

2
12
0 0

[PATCH bpf-next 2/2] selftests/bpf: Test for IPv6 ext header parsing

by Mathieu Jadin

This test creates a client and a server exchanging a single byte with a Segment Routing Header and the eBPF program saves the inner segment in a sk_storage. The test program checks that the segment is correct. Signed-off-by: Mathieu Jadin <mathjadin(a)gmail.com> --- .../bpf/prog_tests/tcp_ipv6_exthdr_srh.c | 171 ++++++++++++++++++ .../selftests/bpf/progs/tcp_ipv6_exthdr_srh.c | 78 ++++++++ 2 files changed, 249 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/tcp_ipv6_exthdr_srh.c create mode 100644 tools/testing/selftests/bpf/progs/tcp_ipv6_exthdr_srh.c diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_ipv6_exthdr_srh.c b/tools/testing/selftests/bpf/prog_tests/tcp_ipv6_exthdr_srh.c new file mode 100644 index 000000000000..70f7ee230975 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tcp_ipv6_exthdr_srh.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <linux/seg6.h> +#include "cgroup_helpers.h" +#include "network_helpers.h" + +struct tcp_srh_storage { + struct in6_addr inner_segment; +}; + +static void send_byte(int fd) +{ + char b = 0x55; + + if (CHECK_FAIL(send(fd, &b, sizeof(b), 0) != 1)) + perror("Failed to send single byte"); +} + +static int verify_srh(int map_fd, int server_fd, struct ipv6_sr_hdr *client_srh) +{ + int err = 0; + struct tcp_srh_storage val; + + if (CHECK_FAIL(bpf_map_lookup_elem(map_fd, &server_fd, &val) < 0)) { + perror("Failed to read socket storage"); + return -1; + } + + if (memcmp(&val.inner_segment, &client_srh->segments[1], + sizeof(struct in6_addr))) { + log_err("The inner segment of the received SRH differs from the sent one"); + err++; + } + + return err; +} + +static int run_test(int cgroup_fd, int listen_fd) +{ + struct bpf_prog_load_attr attr = { + .prog_type = BPF_PROG_TYPE_SOCK_OPS, + .file = "./tcp_ipv6_exthdr_srh.o", + .expected_attach_type = BPF_CGROUP_SOCK_OPS, + }; + size_t srh_size = sizeof(struct ipv6_sr_hdr) + + 2 * sizeof(struct in6_addr); + struct ipv6_sr_hdr *client_srh; + struct bpf_object *obj; + struct bpf_map *map; + struct timeval tv; + int client_fd; + int server_fd; + int prog_fd; + int map_fd; + char byte; + int err; + + err = bpf_prog_load_xattr(&attr, &obj, &prog_fd); + if (err) { + log_err("Failed to load BPF object"); + return -1; + } + + map = bpf_object__next_map(obj, NULL); + map_fd = bpf_map__fd(map); + + err = bpf_prog_attach(prog_fd, cgroup_fd, BPF_CGROUP_SOCK_OPS, 0); + if (err) { + log_err("Failed to attach BPF program"); + goto close_bpf_object; + } + + client_fd = connect_to_fd(listen_fd, 0); + if (client_fd < 0) { + err = -1; + goto close_bpf_object; + } + + server_fd = accept(listen_fd, NULL, 0); + if (server_fd < 0) { + err = -1; + goto close_client_fd; + } + + /* Set an SRH with ::1 as an intermediate segment on the client */ + + client_srh = calloc(1, srh_size); + if (!client_srh) { + log_err("Failed to create the SRH to send"); + goto close_server_fd; + } + client_srh->type = IPV6_SRCRT_TYPE_4; + // We do not count the first 8 bytes (RFC 8200 Section 4.4) + client_srh->hdrlen = (2 * sizeof(struct in6_addr)) >> 3; + client_srh->segments_left = 1; + client_srh->first_segment = 1; + // client_srh->segments[0] is set by the kernel + memcpy(&client_srh->segments[1], &in6addr_loopback, + sizeof(struct in6_addr)); + + if (setsockopt(client_fd, SOL_IPV6, IPV6_RTHDR, client_srh, + srh_size)) { + log_err("Failed to set the SRH on the client"); + goto free_srh; + } + + /* Send traffic with this SRH + * and check its parsing on the server side + */ + + tv.tv_sec = 1; + tv.tv_usec = 0; + if (setsockopt(server_fd, SOL_SOCKET, SO_RCVTIMEO, (const char *)&tv, + sizeof(tv))) { + log_err("Failed to set the receive timeout on the server"); + err = -1; + goto free_srh; + } + + send_byte(client_fd); + if (recv(server_fd, &byte, 1, 0) != 1) { + log_err("Failed to get the byte under one second on the server 2"); + err = -1; + goto free_srh; + } + + err += verify_srh(map_fd, server_fd, client_srh); + +free_srh: + free(client_srh); +close_server_fd: + close(server_fd); +close_client_fd: + close(client_fd); +close_bpf_object: + bpf_object__close(obj); + return err; +} + +void test_tcp_ipv6_exthdr_srh(void) +{ + int server_fd, cgroup_fd; + + cgroup_fd = test__join_cgroup("/tcp_ipv6_exthdr_srh"); + if (CHECK_FAIL(cgroup_fd < 0)) + return; + + server_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0); + if (CHECK_FAIL(server_fd < 0)) + goto close_cgroup_fd; + + if (CHECK_FAIL(system("sysctl net.ipv6.conf.all.seg6_enabled=1"))) + goto close_server; + + if (CHECK_FAIL(system("sysctl net.ipv6.conf.lo.seg6_enabled=1"))) + goto reset_sysctl; + + CHECK_FAIL(run_test(cgroup_fd, server_fd)); + + if (CHECK_FAIL(system("sysctl net.ipv6.conf.lo.seg6_enabled=0"))) + log_err("Cannot reset sysctl net.ipv6.conf.lo.seg6_enabled to 0"); + +reset_sysctl: + if (CHECK_FAIL(system("sysctl net.ipv6.conf.all.seg6_enabled=0"))) + log_err("Cannot reset sysctl net.ipv6.conf.all.seg6_enabled to 0"); + +close_server: + close(server_fd); +close_cgroup_fd: + close(cgroup_fd); +} diff --git a/tools/testing/selftests/bpf/progs/tcp_ipv6_exthdr_srh.c b/tools/testing/selftests/bpf/progs/tcp_ipv6_exthdr_srh.c new file mode 100644 index 000000000000..276bda8bbecb --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tcp_ipv6_exthdr_srh.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/types.h> +#include <bpf/bpf_helpers.h> +#include <linux/in6.h> +#include <linux/ipv6.h> +#include <linux/seg6.h> +#include <linux/bpf.h> + +char _license[] SEC("license") = "GPL"; + +#define NEXTHDR_ROUTING 43 + +struct tcp_srh_storage { + struct in6_addr inner_segment; +}; + +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct tcp_srh_storage); +} socket_storage_map SEC(".maps"); + +/* Check the header received from the active side */ +static int read_incoming_srh(struct bpf_sock_ops *skops, + struct tcp_srh_storage *storage) +{ + __u32 seg_size = 2 * sizeof(struct in6_addr); + struct ipv6_sr_hdr *srh; + struct ipv6hdr *ip6; + void *seg_list; + int ret = 1; + + ip6 = (struct ipv6hdr *)skops->skb_data; + if (ip6 + 1 <= skops->skb_data_end && ip6->nexthdr == NEXTHDR_ROUTING) { + srh = (struct ipv6_sr_hdr *)(ip6 + 1); + if (srh + 1 <= skops->skb_data_end) { + if (srh->type != IPV6_SRCRT_TYPE_4) + return ret; + + seg_list = (void *)(srh + 1); + if (seg_list + seg_size <= skops->skb_data_end) { + // This is an SRH with at least 2 segments + storage->inner_segment = srh->segments[1]; + ret = 0; + } + } + } + + return ret; +} + +SEC("sockops") +int srh_read(struct bpf_sock_ops *skops) +{ + struct tcp_srh_storage *storage; + int true_val = 1; + + if (!skops->sk) + return 1; + + storage = bpf_sk_storage_get(&socket_storage_map, skops->sk, 0, + BPF_SK_STORAGE_GET_F_CREATE); + if (!storage) + return 1; + + switch (skops->op) { + case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB: + case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB: + bpf_sock_ops_cb_flags_set(skops, skops->bpf_sock_ops_cb_flags | + BPF_SOCK_OPS_PARSE_IPV6_HDR_CB_FLAG); + break; + case BPF_SOCK_OPS_PARSE_IPV6_HDR_CB: + return read_incoming_srh(skops, storage); + } + + return 0; +} -- 2.32.0

4 years

1
0
0 0

selftests/net/fcnal-test.sh: ipv6_bind failed

by Zhou, Jie2X

hi, I test ipv6_bind by "fcnal-test.sh -v -t ipv6_bind" in kernel v5.16-rc3. There are two tests failed. TEST: TCP socket bind to out of scope local address - ns-A loopback IPv6 [FAIL] TEST: TCP socket bind to VRF address with device bind - VRF IPv6 [FAIL] In fcnal-test.sh expected command error not occurred. ipv6_addr_bind_novrf() { ...... log_test_addr ${a} $? 1 "TCP socket bind to out of scope local address" ipv6_addr_bind_vrf() { ...... log_test_addr ${a} $? 1 "TCP socket bind to VRF address with device bind" Did I set something wrong that result in these failed? The test output is attached. best regards,

4 years

1
0
0 0

2025

2024

2023

2022

2021

2020

2019

2018

2017

Linux-kselftest-mirror