Re: [PATCH] kunit: tool: don't include KTAP headers and the like in the test log

On Tue, Nov 29, 2022 at 8:12 AM Daniel Latypov dlatypov@google.com wrote:

We print the "test log" on failure. This is meant to be all the kernel output that happened during the test.

But we also include the special KTAP lines in it, which are often redundant.

E.g. we include the "not ok" line in the log, right before we print that the test case failed... [13:51:48] Expected 2 + 1 == 2, but [13:51:48] 2 + 1 == 3 (0x3) [13:51:48] not ok 1 example_simple_test [13:51:48] [FAILED] example_simple_test

More full example after this patch: [13:51:48] =================== example (4 subtests) =================== [13:51:48] # example_simple_test: initializing [13:51:48] # example_simple_test: EXPECTATION FAILED at lib/kunit/kunit-example-test.c:29 [13:51:48] Expected 2 + 1 == 2, but [13:51:48] 2 + 1 == 3 (0x3) [13:51:48] [FAILED] example_simple_test

Signed-off-by: Daniel Latypov dlatypov@google.com

I totally agree we should skip these from the log. (Unless --raw_output is enabled, but that obviously doesn't apply.)

Going forward, I think we should also probably disable kunit.stats_enabled when running via kunit.py, too (again, unless --raw_output is used.)

In any case, this looks good and works well here.

Reviewed-by: David Gow davidgow@google.com

Cheers, -- David

tools/testing/kunit/kunit_parser.py | 8 ++++---- tools/testing/kunit/kunit_tool_test.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 4cc2f8b7ecd0..99b8f058db40 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -295,7 +295,7 @@ def parse_ktap_header(lines: LineStream, test: Test) -> bool: check_version(version_num, TAP_VERSIONS, 'TAP', test) else: return False

•   test.log.append(lines.pop())
  

•   lines.pop()
    return True
  
  

TEST_HEADER = re.compile(r'^# Subtest: (.*)$') @@ -318,8 +318,8 @@ def parse_test_header(lines: LineStream, test: Test) -> bool: match = TEST_HEADER.match(lines.peek()) if not match: return False

•   test.log.append(lines.pop())
    test.name = match.group(1)
  

•   lines.pop()
    return True
  
  

TEST_PLAN = re.compile(r'1..([0-9]+)') @@ -345,9 +345,9 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: if not match: test.expected_count = None return False

•   test.log.append(lines.pop())
    expected_count = int(match.group(1))
    test.expected_count = expected_count
  

•   lines.pop()
    return True
  
  

TEST_RESULT = re.compile(r'^(ok|not ok) ([0-9]+) (- )?([^#]*)( # .*)?$') @@ -409,7 +409,7 @@ def parse_test_result(lines: LineStream, test: Test, # Check if line matches test result line format if not match: return False

•   test.log.append(lines.pop())
  

•   lines.pop()
  
    # Set name of test object
    if skip_match:
  

diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index d7f669cbf2a8..1ef921ac4331 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -84,6 +84,10 @@ class KUnitParserTest(unittest.TestCase): self.print_mock = mock.patch('kunit_printer.Printer.print').start() self.addCleanup(mock.patch.stopall)

•   def noPrintCallContains(self, substr: str):
  

•           for call in self.print_mock.mock_calls:
  

•                   self.assertNotIn(substr, call.args[0])
  

•   def assertContains(self, needle: str, haystack: kunit_parser.LineStream):
            # Clone the iterator so we can print the contents on failure.
            copy, backup = itertools.tee(haystack)
  

@@ -327,6 +331,19 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests(file.readlines()) self.print_mock.assert_any_call(StrContains('suite (1 subtest)'))

•   def test_show_test_output_on_failure(self):
  

•           output = """
  

•           KTAP version 1
  

•           1..1
  

•             Test output.
  

•           not ok 1 test1
  

•           """
  

•           result = kunit_parser.parse_run_tests(output.splitlines())
  

•           self.assertEqual(kunit_parser.TestStatus.FAILURE, result.status)
  

•           self.print_mock.assert_any_call(StrContains('Test output.'))
  

•           self.noPrintCallContains('not ok 1 test1')
  

def line_stream_from_strs(strs: Iterable[str]) -> kunit_parser.LineStream: return kunit_parser.LineStream(enumerate(strs, start=1))

base-commit: 11300092f6f4dc4103ac4bd950d62f94effc736a

2.38.1.584.g0f3c55d4c2-goog