Re: [PATCH 2/3] kunit: tool: unit tests all check parser errors, standardize formatting a bit

On Thu, Nov 3, 2022 at 12:40 AM 'Daniel Latypov' via KUnit Development kunit-dev@googlegroups.com wrote:

Let's verify that the parser isn't reporting any errors for valid inputs.

This change also

• does result.status checking on one line
• makes sure we consistently do it outside of the `with` block

Signed-off-by: Daniel Latypov dlatypov@google.com

Looks good, thanks.

Note that this patch does conflict with "kunit: tool: print summary of failed tests if a few failed out of a lot": https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git/co... It's only a context line issue, though.

Reviewed-by: David Gow davidgow@google.com

Cheers, -- David

tools/testing/kunit/kunit_tool_test.py | 93 +++++++++++--------------- 1 file changed, 38 insertions(+), 55 deletions(-)

diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 9fa4babb2506..0063773c0fc4 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -133,33 +133,29 @@ class KUnitParserTest(unittest.TestCase): all_passed_log = test_data_path('test_is_test_passed-all_passed.log') with open(all_passed_log) as file: result = kunit_parser.parse_run_tests(file.readlines())

•           self.assertEqual(
  

•                   kunit_parser.TestStatus.SUCCESS,
  

•                   result.status)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
  

•           self.assertEqual(result.counts.errors, 0)
  
    def test_parse_successful_nested_tests_log(self):
            all_passed_log = test_data_path('test_is_test_passed-all_passed_nested.log')
            with open(all_passed_log) as file:
                    result = kunit_parser.parse_run_tests(file.readlines())
  

•           self.assertEqual(
  

•                   kunit_parser.TestStatus.SUCCESS,
  

•                   result.status)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
  

•           self.assertEqual(result.counts.errors, 0)
  
    def test_kselftest_nested(self):
            kselftest_log = test_data_path('test_is_test_passed-kselftest.log')
            with open(kselftest_log) as file:
                    result = kunit_parser.parse_run_tests(file.readlines())
  

•                   self.assertEqual(
  

•                           kunit_parser.TestStatus.SUCCESS,
  

•                           result.status)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
  

•           self.assertEqual(result.counts.errors, 0)
  
    def test_parse_failed_test_log(self):
            failed_log = test_data_path('test_is_test_passed-failure.log')
            with open(failed_log) as file:
                    result = kunit_parser.parse_run_tests(file.readlines())
  

•           self.assertEqual(
  

•                   kunit_parser.TestStatus.FAILURE,
  

•                   result.status)
  

•           self.assertEqual(kunit_parser.TestStatus.FAILURE, result.status)
  

•           self.assertEqual(result.counts.errors, 0)
  
    def test_no_header(self):
            empty_log = test_data_path('test_is_test_passed-no_tests_run_no_header.log')
  

@@ -167,9 +163,8 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines(file.readlines())) self.assertEqual(0, len(result.subtests))

•           self.assertEqual(
  

•                   kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS,
  

•                   result.status)
  

•           self.assertEqual(kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS, result.status)
  

•           self.assertEqual(result.counts.errors, 1)
  
    def test_missing_test_plan(self):
            missing_plan_log = test_data_path('test_is_test_passed-'
  

@@ -180,9 +175,7 @@ class KUnitParserTest(unittest.TestCase): file.readlines())) # A missing test plan is not an error. self.assertEqual(result.counts, kunit_parser.TestCounts(passed=10, errors=0))

•           self.assertEqual(
  

•                   kunit_parser.TestStatus.SUCCESS,
  

•                   result.status)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
  
    def test_no_tests(self):
            header_log = test_data_path('test_is_test_passed-no_tests_run_with_header.log')
  

@@ -190,9 +183,8 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines(file.readlines())) self.assertEqual(0, len(result.subtests))

•           self.assertEqual(
  

•                   kunit_parser.TestStatus.NO_TESTS,
  

•                   result.status)
  

•           self.assertEqual(kunit_parser.TestStatus.NO_TESTS, result.status)
  

•           self.assertEqual(result.counts.errors, 1)
  
    def test_no_tests_no_plan(self):
            no_plan_log = test_data_path('test_is_test_passed-no_tests_no_plan.log')
  

@@ -203,7 +195,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual( kunit_parser.TestStatus.NO_TESTS, result.subtests[0].subtests[0].status)

•           self.assertEqual(1, result.counts.errors)
  

•           self.assertEqual(result.counts, kunit_parser.TestCounts(passed=1, errors=1))
  
  
    def test_no_kunit_output(self):
  

@@ -215,6 +207,7 @@ class KUnitParserTest(unittest.TestCase): print_mock.assert_any_call(StrContains('could not find any KTAP output!')) print_mock.stop() self.assertEqual(0, len(result.subtests))

•           self.assertEqual(result.counts.errors, 1)
  
    def test_skipped_test(self):
            skipped_log = test_data_path('test_skip_tests.log')
  

@@ -222,18 +215,16 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests(file.readlines())

            # A skipped test does not fail the whole suite.

•           self.assertEqual(
  

•                   kunit_parser.TestStatus.SUCCESS,
  

•                   result.status)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
  

•           self.assertEqual(result.counts, kunit_parser.TestCounts(passed=4, skipped=1))
  
    def test_skipped_all_tests(self):
            skipped_log = test_data_path('test_skip_all_tests.log')
            with open(skipped_log) as file:
                    result = kunit_parser.parse_run_tests(file.readlines())
  
  

•           self.assertEqual(
  

•                   kunit_parser.TestStatus.SKIPPED,
  

•                   result.status)
  

•           self.assertEqual(kunit_parser.TestStatus.SKIPPED, result.status)
  

•           self.assertEqual(result.counts, kunit_parser.TestCounts(skipped=5))
  
    def test_ignores_hyphen(self):
            hyphen_log = test_data_path('test_strip_hyphen.log')
  

@@ -241,9 +232,7 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests(file.readlines())

            # A skipped test does not fail the whole suite.

•           self.assertEqual(
  

•                   kunit_parser.TestStatus.SUCCESS,
  

•                   result.status)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
            self.assertEqual(
                    "sysctl_test",
                    result.subtests[0].name)
  

@@ -257,55 +246,49 @@ class KUnitParserTest(unittest.TestCase): prefix_log = test_data_path('test_config_printk_time.log') with open(prefix_log) as file: result = kunit_parser.parse_run_tests(file.readlines())

•                   self.assertEqual(
  

•                           kunit_parser.TestStatus.SUCCESS,
  

•                           result.status)
  

•                   self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
  

•           self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(result.counts.errors, 0)
  
    def test_ignores_multiple_prefixes(self):
            prefix_log = test_data_path('test_multiple_prefixes.log')
            with open(prefix_log) as file:
                    result = kunit_parser.parse_run_tests(file.readlines())
  

•                   self.assertEqual(
  

•                           kunit_parser.TestStatus.SUCCESS,
  

•                           result.status)
  

•                   self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
  

•           self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(result.counts.errors, 0)
  
    def test_prefix_mixed_kernel_output(self):
            mixed_prefix_log = test_data_path('test_interrupted_tap_output.log')
            with open(mixed_prefix_log) as file:
                    result = kunit_parser.parse_run_tests(file.readlines())
  

•                   self.assertEqual(
  

•                           kunit_parser.TestStatus.SUCCESS,
  

•                           result.status)
  

•                   self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
  

•           self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(result.counts.errors, 0)
  
    def test_prefix_poundsign(self):
            pound_log = test_data_path('test_pound_sign.log')
            with open(pound_log) as file:
                    result = kunit_parser.parse_run_tests(file.readlines())
  

•                   self.assertEqual(
  

•                           kunit_parser.TestStatus.SUCCESS,
  

•                           result.status)
  

•                   self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
  

•           self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(result.counts.errors, 0)
  
    def test_kernel_panic_end(self):
            panic_log = test_data_path('test_kernel_panic_interrupt.log')
            with open(panic_log) as file:
                    result = kunit_parser.parse_run_tests(file.readlines())
  

•                   self.assertEqual(
  

•                           kunit_parser.TestStatus.TEST_CRASHED,
  

•                           result.status)
  

•                   self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(kunit_parser.TestStatus.TEST_CRASHED, result.status)
  

•           self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertGreaterEqual(result.counts.errors, 1)
  
    def test_pound_no_prefix(self):
            pound_log = test_data_path('test_pound_no_prefix.log')
            with open(pound_log) as file:
                    result = kunit_parser.parse_run_tests(file.readlines())
  

•                   self.assertEqual(
  

•                           kunit_parser.TestStatus.SUCCESS,
  

•                           result.status)
  

•                   self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
  

•           self.assertEqual('kunit-resource-test', result.subtests[0].name)
  

•           self.assertEqual(result.counts.errors, 0)
  
  

def line_stream_from_strs(strs: Iterable[str]) -> kunit_parser.LineStream: return kunit_parser.LineStream(enumerate(strs, start=1)) -- 2.38.1.273.g43a17bfeac-goog

-- You received this message because you are subscribed to the Google Groups "KUnit Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to kunit-dev+unsubscribe@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/kunit-dev/20221102164005.2516646-2-dlatypo....