RE: [PATCH v1 2/2] kunit: tool: allow generating test results in JSON

-----Original Message----- From: Brendan Higgins Sent: Friday, August 7, 2020 7:17 PM

From: Heidi Fahim heidifahim@google.com

Add a --json flag, which when specified when kunit_tool is run, generates JSON formatted test results conforming to the KernelCI API test_group spec[1]. The user can the new flag to specify a filename as

Seems to be missing a word. "The user can ? the new flag"

the value to json in order to store the JSON results under linux/.

Link[1]: https://api.kernelci.org/schema-test-group.html#post Signed-off-by: Heidi Fahim heidifahim@google.com Signed-off-by: Brendan Higgins brendanhiggins@google.com

---

tools/testing/kunit/kunit.py | 35 +++++++++++--- tools/testing/kunit/kunit_json.py | 63 ++++++++++++++++++++++++++ tools/testing/kunit/kunit_tool_test.py | 33 ++++++++++++++ 3 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 tools/testing/kunit/kunit_json.py

diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 96344a11ff1f..485b7c63b967 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -17,6 +17,7 @@ from collections import namedtuple from enum import Enum, auto

import kunit_config +import kunit_json import kunit_kernel import kunit_parser

@@ -30,9 +31,9 @@ KunitBuildRequest = namedtuple('KunitBuildRequest', KunitExecRequest = namedtuple('KunitExecRequest', ['timeout', 'build_dir', 'alltests']) KunitParseRequest = namedtuple('KunitParseRequest',

• 	       ['raw_output', 'input_data'])
  

• 	       ['raw_output', 'input_data', 'build_dir', 'json'])
  

KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs',

• 			   'build_dir', 'alltests',
  

• 			   'build_dir', 'alltests', 'json',
  		   'make_options'])
  
  

KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] @@ -113,12 +114,22 @@ def parse_tests(request: KunitParseRequest) -> KunitResult: test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, [], 'Tests not Parsed.')

• if request.raw_output: kunit_parser.raw_output(request.input_data) else: test_result = kunit_parser.parse_run_tests(request.input_data) parse_end = time.time()

• if request.json:

• json_obj = kunit_json.get_json_result(
  

• 			test_result=test_result,
  

• 			def_config='kunit_defconfig',
  

• 			build_dir=request.build_dir,
  

• 			json_path=request.json)
  

• if request.json == 'stdout':
  

• 	print(json_obj)
  

• if test_result.status != kunit_parser.TestStatus.SUCCESS: return KunitResult(KunitStatus.TEST_FAILURE, test_result, parse_end - parse_start)

@@ -151,7 +162,9 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, return exec_result

parse_request = KunitParseRequest(request.raw_output,

• 			  exec_result.result)
  

• 			  exec_result.result,
  

• 			  request.build_dir,
  

• 			  request.json)
  

  parse_result = parse_tests(parse_request)

  run_end = time.time()

@@ -195,7 +208,12 @@ def add_exec_opts(parser): def add_parse_opts(parser): parser.add_argument('--raw_output', help='don't format output from kernel', action='store_true')

• parser.add_argument('--json',

• 	    nargs='?',
  

• 	    help='Stores test results in a JSON, and either '
  

• 	    'prints to stdout or saves to file if a '
  

• 	    'filename is specified',
  

• 	    type=str, const='stdout', default=None)
  
  

def main(argv, linux=None): parser = argparse.ArgumentParser( @@ -254,6 +272,7 @@ def main(argv, linux=None): cli_args.jobs, cli_args.build_dir, cli_args.alltests,

• 		       cli_args.json,
  	       cli_args.make_options)
  

  result = run_tests(linux, request) if result.status != KunitStatus.SUCCESS:

@@ -308,7 +327,9 @@ def main(argv, linux=None): cli_args.alltests) exec_result = exec_tests(linux, exec_request) parse_request = KunitParseRequest(cli_args.raw_output,

• 				  exec_result.result)
  

• 				  exec_result.result,
  

• 				  cli_args.build_dir,
  

• 				  cli_args.json)
  

  result = parse_tests(parse_request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % (

@@ -322,7 +343,9 @@ def main(argv, linux=None): with open(cli_args.file, 'r') as f: kunit_output = f.read().splitlines() request = KunitParseRequest(cli_args.raw_output,

• 			    kunit_output)
  

• 			    kunit_output,
  

• 			    cli_args.build_dir,
  

• 			    cli_args.json)
  

  result = parse_tests(request) if result.status != KunitStatus.SUCCESS: sys.exit(1)

diff --git a/tools/testing/kunit/kunit_json.py b/tools/testing/kunit/kunit_json.py new file mode 100644 index 000000000000..624b31b2dbd6 --- /dev/null +++ b/tools/testing/kunit/kunit_json.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Generates JSON from KUnit results according to +# KernelCI spec: https://github.com/kernelci/kernelci-doc/wiki/Test-API +# +# Copyright (C) 2020, Google LLC. +# Author: Heidi Fahim heidifahim@google.com

+import json +import os

+import kunit_parser

+from kunit_parser import TestStatus

+def get_json_result(test_result, def_config, build_dir, json_path):

• sub_groups = []
• # Each test suite is mapped to a KernelCI sub_group
• for test_suite in test_result.suites:

• sub_group = {
  

• 	"name": test_suite.name,
  

• 	"arch": "UM",
  

• 	"defconfig": def_config,
  

• 	"build_environment": build_dir,
  

• 	"test_cases": [],
  

• 	"lab_name": None,
  

• 	"kernel": None,
  

• 	"job": None,
  

• 	"git_branch": "kselftest",
  

• }
  

• test_cases = []
  

• # TODO: Add attachments attribute in test_case with detailed
  

• #  failure message, see https://api.kernelci.org/schema-test-case.html#get
  

• for case in test_suite.cases:
  

• 	test_case = {"name": case.name, "status": "FAIL"}
  

• 	if case.status == TestStatus.SUCCESS:
  

• 		test_case["status"] = "PASS"
  

• 	elif case.status == TestStatus.TEST_CRASHED:
  

• 		test_case["status"] = "ERROR"
  

• 	test_cases.append(test_case)
  

• sub_group["test_cases"] = test_cases
  

• sub_groups.append(sub_group)
  

• test_group = {

• "name": "KUnit Test Group",
  

• "arch": "UM",
  

• "defconfig": def_config,
  

• "build_environment": build_dir,
  

• "sub_groups": sub_groups,
  

• "lab_name": None,
  

• "kernel": None,
  

• "job": None,
  

• "git_branch": "kselftest",
  

• }
• json_obj = json.dumps(test_group, indent=4)
• if json_path != 'stdout':

• with open(json_path, 'w') as result_path:
  

• 	result_path.write(json_obj)
  

• root = __file__.split('tools/testing/kunit/')[0]
  

• kunit_parser.print_with_timestamp(
  

• 	"Test results stored in %s" %
  

• 	os.path.join(root, result_path.name))
  

• return json_obj

diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 287c74d821c3..99c3c5671ea4 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -11,11 +11,13 @@ from unittest import mock

import tempfile, shutil # Handling test_tmpdir

+import json import os

import kunit_config import kunit_parser import kunit_kernel +import kunit_json import kunit

test_tmpdir = '' @@ -230,6 +232,37 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests(file.readlines()) self.assertEqual('kunit-resource-test', result.suites[0].name)

+class KUnitJsonTest(unittest.TestCase):

• def _json_for(self, log_file):

• with(open(get_absolute_path(log_file))) as file:
  

• 	test_result = kunit_parser.parse_run_tests(file)
  

• 	json_obj = kunit_json.get_json_result(
  

• 		test_result=test_result,
  

• 		def_config='kunit_defconfig',
  

• 		build_dir=None,
  

• 		json_path='stdout')
  

• return json.loads(json_obj)
  

• def test_failed_test_json(self):

• result = self._json_for(
  

• 	'test_data/test_is_test_passed-failure.log')
  

• self.assertEqual(
  

• 	{'name': 'example_simple_test', 'status': 'FAIL'},
  

• 	result["sub_groups"][1]["test_cases"][0])
  

• def test_crashed_test_json(self):

• result = self._json_for(
  

• 	'test_data/test_is_test_passed-crash.log')
  

• self.assertEqual(
  

• 	{'name': 'example_simple_test', 'status': 'ERROR'},
  

• 	result["sub_groups"][1]["test_cases"][0])
  

• def test_no_tests_json(self):

• result = self._json_for(
  

• 	'test_data/test_is_test_passed-no_tests_run.log')
  

• self.assertEqual(0, len(result['sub_groups']))
  

class StrContains(str): def __eq__(self, other): return self in other -- 2.28.0.236.gb10cc79966-goog