-----Original Message----- From: Brendan Higgins Sent: Friday, August 7, 2020 7:17 PM

From: Heidi Fahim heidifahim@google.com

Add a --json flag, which when specified when kunit_tool is run, generates JSON formatted test results conforming to the KernelCI API test_group spec[1]. The user can the new flag to specify a filename as

Seems to be missing a word. "The user can ? the new flag"

the value to json in order to store the JSON results under linux/.

Link[1]: https://api.kernelci.org/schema-test-group.html#post Signed-off-by: Heidi Fahim heidifahim@google.com Signed-off-by: Brendan Higgins brendanhiggins@google.com

---

tools/testing/kunit/kunit.py | 35 +++++++++++--- tools/testing/kunit/kunit_json.py | 63 ++++++++++++++++++++++++++ tools/testing/kunit/kunit_tool_test.py | 33 ++++++++++++++ 3 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 tools/testing/kunit/kunit_json.py

diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 96344a11ff1f..485b7c63b967 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -17,6 +17,7 @@ from collections import namedtuple from enum import Enum, auto

import kunit_config +import kunit_json import kunit_kernel import kunit_parser

@@ -30,9 +31,9 @@ KunitBuildRequest = namedtuple('KunitBuildRequest', KunitExecRequest = namedtuple('KunitExecRequest', ['timeout', 'build_dir', 'alltests']) KunitParseRequest = namedtuple('KunitParseRequest',

• 	       ['raw_output', 'input_data'])
  

• 	       ['raw_output', 'input_data', 'build_dir', 'json'])
  

KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs',

• 			   'build_dir', 'alltests',
  

• 			   'build_dir', 'alltests', 'json',
  		   'make_options'])
  
  

KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] @@ -113,12 +114,22 @@ def parse_tests(request: KunitParseRequest) -> KunitResult: test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, [], 'Tests not Parsed.')

• if request.raw_output: kunit_parser.raw_output(request.input_data) else: test_result = kunit_parser.parse_run_tests(request.input_data) parse_end = time.time()

• if request.json:

• json_obj = kunit_json.get_json_result(
  

• 			test_result=test_result,
  

• 			def_config='kunit_defconfig',
  

• 			build_dir=request.build_dir,
  

• 			json_path=request.json)
  

• if request.json == 'stdout':
  

• 	print(json_obj)
  

• if test_result.status != kunit_parser.TestStatus.SUCCESS: return KunitResult(KunitStatus.TEST_FAILURE, test_result, parse_end - parse_start)

@@ -151,7 +162,9 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, return exec_result

parse_request = KunitParseRequest(request.raw_output,

• 			  exec_result.result)
  

• 			  exec_result.result,
  

• 			  request.build_dir,
  

• 			  request.json)
  

  parse_result = parse_tests(parse_request)

  run_end = time.time()

@@ -195,7 +208,12 @@ def add_exec_opts(parser): def add_parse_opts(parser): parser.add_argument('--raw_output', help='don't format output from kernel', action='store_true')

• parser.add_argument('--json',

• 	    nargs='?',
  

• 	    help='Stores test results in a JSON, and either '
  

• 	    'prints to stdout or saves to file if a '
  

• 	    'filename is specified',
  

• 	    type=str, const='stdout', default=None)
  
  

def main(argv, linux=None): parser = argparse.ArgumentParser( @@ -254,6 +272,7 @@ def main(argv, linux=None): cli_args.jobs, cli_args.build_dir, cli_args.alltests,

• 		       cli_args.json,
  	       cli_args.make_options)
  

  result = run_tests(linux, request) if result.status != KunitStatus.SUCCESS:

@@ -308,7 +327,9 @@ def main(argv, linux=None): cli_args.alltests) exec_result = exec_tests(linux, exec_request) parse_request = KunitParseRequest(cli_args.raw_output,

• 				  exec_result.result)
  

• 				  exec_result.result,
  

• 				  cli_args.build_dir,
  

• 				  cli_args.json)
  

  result = parse_tests(parse_request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % (

@@ -322,7 +343,9 @@ def main(argv, linux=None): with open(cli_args.file, 'r') as f: kunit_output = f.read().splitlines() request = KunitParseRequest(cli_args.raw_output,

• 			    kunit_output)
  

• 			    kunit_output,
  

• 			    cli_args.build_dir,
  

• 			    cli_args.json)
  

  result = parse_tests(request) if result.status != KunitStatus.SUCCESS: sys.exit(1)

diff --git a/tools/testing/kunit/kunit_json.py b/tools/testing/kunit/kunit_json.py new file mode 100644 index 000000000000..624b31b2dbd6 --- /dev/null +++ b/tools/testing/kunit/kunit_json.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Generates JSON from KUnit results according to +# KernelCI spec: https://github.com/kernelci/kernelci-doc/wiki/Test-API +# +# Copyright (C) 2020, Google LLC. +# Author: Heidi Fahim heidifahim@google.com

+import json +import os

+import kunit_parser

+from kunit_parser import TestStatus

+def get_json_result(test_result, def_config, build_dir, json_path):

• sub_groups = []
• # Each test suite is mapped to a KernelCI sub_group
• for test_suite in test_result.suites:

• sub_group = {
  

• 	"name": test_suite.name,
  

• 	"arch": "UM",
  

• 	"defconfig": def_config,
  

• 	"build_environment": build_dir,
  

• 	"test_cases": [],
  

• 	"lab_name": None,
  

• 	"kernel": None,
  

• 	"job": None,
  

• 	"git_branch": "kselftest",
  

• }
  

• test_cases = []
  

• # TODO: Add attachments attribute in test_case with detailed
  

• #  failure message, see https://api.kernelci.org/schema-test-case.html#get
  

• for case in test_suite.cases:
  

• 	test_case = {"name": case.name, "status": "FAIL"}
  

• 	if case.status == TestStatus.SUCCESS:
  

• 		test_case["status"] = "PASS"
  

• 	elif case.status == TestStatus.TEST_CRASHED:
  

• 		test_case["status"] = "ERROR"
  

• 	test_cases.append(test_case)
  

• sub_group["test_cases"] = test_cases
  

• sub_groups.append(sub_group)
  

• test_group = {

• "name": "KUnit Test Group",
  

• "arch": "UM",
  

• "defconfig": def_config,
  

• "build_environment": build_dir,
  

• "sub_groups": sub_groups,
  

• "lab_name": None,
  

• "kernel": None,
  

• "job": None,
  

• "git_branch": "kselftest",
  

• }
• json_obj = json.dumps(test_group, indent=4)
• if json_path != 'stdout':

• with open(json_path, 'w') as result_path:
  

• 	result_path.write(json_obj)
  

• root = __file__.split('tools/testing/kunit/')[0]
  

• kunit_parser.print_with_timestamp(
  

• 	"Test results stored in %s" %
  

• 	os.path.join(root, result_path.name))
  

• return json_obj

diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 287c74d821c3..99c3c5671ea4 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -11,11 +11,13 @@ from unittest import mock

import tempfile, shutil # Handling test_tmpdir

+import json import os

import kunit_config import kunit_parser import kunit_kernel +import kunit_json import kunit

test_tmpdir = '' @@ -230,6 +232,37 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests(file.readlines()) self.assertEqual('kunit-resource-test', result.suites[0].name)

+class KUnitJsonTest(unittest.TestCase):

• def _json_for(self, log_file):

• with(open(get_absolute_path(log_file))) as file:
  

• 	test_result = kunit_parser.parse_run_tests(file)
  

• 	json_obj = kunit_json.get_json_result(
  

• 		test_result=test_result,
  

• 		def_config='kunit_defconfig',
  

• 		build_dir=None,
  

• 		json_path='stdout')
  

• return json.loads(json_obj)
  

• def test_failed_test_json(self):

• result = self._json_for(
  

• 	'test_data/test_is_test_passed-failure.log')
  

• self.assertEqual(
  

• 	{'name': 'example_simple_test', 'status': 'FAIL'},
  

• 	result["sub_groups"][1]["test_cases"][0])
  

• def test_crashed_test_json(self):

• result = self._json_for(
  

• 	'test_data/test_is_test_passed-crash.log')
  

• self.assertEqual(
  

• 	{'name': 'example_simple_test', 'status': 'ERROR'},
  

• 	result["sub_groups"][1]["test_cases"][0])
  

• def test_no_tests_json(self):

• result = self._json_for(
  

• 	'test_data/test_is_test_passed-no_tests_run.log')
  

• self.assertEqual(0, len(result['sub_groups']))
  

class StrContains(str): def __eq__(self, other): return self in other -- 2.28.0.236.gb10cc79966-goog

On Sat, Aug 8, 2020 at 4:51 PM Brendan Higgins brendanhiggins@google.com wrote:

On Fri, Aug 7, 2020 at 10:45 PM David Gow davidgow@google.com wrote:

...

On Sat, Aug 8, 2020 at 9:17 AM Brendan Higgins brendanhiggins@google.com wrote:

...

Currently kunit_tool does not work correctly when executed from a path outside of the kernel tree, so make sure that the current working directory is correct and the kunit_dir is properly initialized before running.

Signed-off-by: Brendan Higgins brendanhiggins@google.com

tools/testing/kunit/kunit.py | 8 ++++++++ 1 file changed, 8 insertions(+)

diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 425ef40067e7..96344a11ff1f 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -237,9 +237,14 @@ def main(argv, linux=None):

    cli_args = parser.parse_args(argv)

•   if get_kernel_root_path():
  

•           print('cd ' + get_kernel_root_path())
  

Do we want to print this, or is it a leftover debug statement?

Whoops, I was supposed to delete that. That's embarrassing... ^_^;

...

...

•           os.chdir(get_kernel_root_path())
  

•   if cli_args.subcommand == 'run':
            if not os.path.exists(cli_args.build_dir):
                    os.mkdir(cli_args.build_dir)
  

•                   create_default_kunitconfig()
  

Why are we adding this everywhere when it's already in config_tests, which should already be called in all of the places where a kunitconfig is required?

Ah yes, .kunitconfig needs to be created before config_tests() can be called because the LinuxSourceTree constructor needs .kunitconfig to exist.

I see. I guess the ultimate solution will be for LinuxSourceTree not require a .kunitconfig unless it's actually being used.

...

Is the goal to always copy the default kunitconfig when creating a new build_dir? While I can sort-of see why we might want to do that, if the build dir doesn't exist, most of the subcommands will fail anyway (maybe we should only create the build-dir for 'config' and 'run'?)

I just did it because we were getting a failure in a constructor so we couldn't do much. Ideally we would check that the current state allows for the command that the user intended to run, but I think that's beyond the scope of this change.

So I guess the real question is: Is it okay for it to crash in the constructor with a cryptic error message for now, or do we want to let it fail with a slightly less cryptic message later?

I personally am leaning towards allowing it to crash in the build, exec, etc. subcommands for now, and tidying up the error messages later, rather than silently creating a blank build dir, only for it then to fail later.

In the meantime, yeah, we can add this for the config and run tasks, and maybe remove the whole "if cli_args.build_dir" / mkdir branch from the other subcommands.

If we weren't going to fix the LinuxSourceTree constructor, it'd make sense to get rid of the redundant code to create it in config_tests(), too, but I'm not sure it's worthwhile.

In any case, now I know what's happening, I'm okay with anything moderately sensible which gets the 'config' and 'run' subcommands working on an empty build dir, and the code and error messages can be fixed when tidying up the LinuxSourceTree() constructor in a separate patch.

Cheers, -- David

...

...

            if not linux:
                    linux = kunit_kernel.LinuxSourceTree()

@@ -257,6 +262,7 @@ def main(argv, linux=None): if cli_args.build_dir: if not os.path.exists(cli_args.build_dir): os.mkdir(cli_args.build_dir)

•                           create_default_kunitconfig()
  
            if not linux:
                    linux = kunit_kernel.LinuxSourceTree()
  

@@ -273,6 +279,7 @@ def main(argv, linux=None): if cli_args.build_dir: if not os.path.exists(cli_args.build_dir): os.mkdir(cli_args.build_dir)

•                           create_default_kunitconfig()
  
            if not linux:
                    linux = kunit_kernel.LinuxSourceTree()
  

@@ -291,6 +298,7 @@ def main(argv, linux=None): if cli_args.build_dir: if not os.path.exists(cli_args.build_dir): os.mkdir(cli_args.build_dir)

•                           create_default_kunitconfig()
  
            if not linux:
                    linux = kunit_kernel.LinuxSourceTree()
  
  

base-commit: 30185b69a2d533c4ba6ca926b8390ce7de495e29

2.28.0.236.gb10cc79966-goog