Add an extension at Sphinx to allow documenting tests. Such extension uses a class developed for me to document GPU tests as part of the IGT test suite: https://gitlab.freedesktop.org/drm/igt-gpu-tools

The idea is to keep the test_list.py in sync on both trees, while adding any kernel-specific logic inside test_kdoc.py.

Signed-off-by: Mauro Carvalho Chehab mchehab@kernel.org ---

To avoid mailbombing on a large number of people, only mailing lists were C/C on the cover. See [PATCH RFC 0/2] at: https://lore.kernel.org/all/cover.1689171160.git.mchehab@kernel.org/

Documentation/conf.py | 2 +- Documentation/sphinx/test_kdoc.py | 108 +++ Documentation/sphinx/test_list.py | 1288 +++++++++++++++++++++++++++++ 3 files changed, 1397 insertions(+), 1 deletion(-) create mode 100644 Documentation/sphinx/test_kdoc.py create mode 100644 Documentation/sphinx/test_list.py

diff --git a/Documentation/conf.py b/Documentation/conf.py index d4fdf6a3875a..cec017a6cab5 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -55,7 +55,7 @@ needs_sphinx = '1.7' extensions = ['kerneldoc', 'rstFlatTable', 'kernel_include', 'kfigure', 'sphinx.ext.ifconfig', 'automarkup', 'maintainers_include', 'sphinx.ext.autosectionlabel', - 'kernel_abi', 'kernel_feat'] + 'kernel_abi', 'kernel_feat', 'test_kdoc']

if major >= 3: if (major > 3) or (minor > 0 or patch >= 2): diff --git a/Documentation/sphinx/test_kdoc.py b/Documentation/sphinx/test_kdoc.py new file mode 100644 index 000000000000..41357438c5fd --- /dev/null +++ b/Documentation/sphinx/test_kdoc.py @@ -0,0 +1,108 @@ +# coding=utf-8 +# SPDX-License-Identifier: GPL-2.0 +# +u""" + "include-test + ~~~~~~~~~~~~~ + + Implementation of the ``include-test`` reST-directive. + + :copyright: Copyright (C) 2023 Intel Corporation + :author: Mauro Carvalho Chehab + :maintained-by: Mauro Carvalho Chehab mchehab@kernel.org + :license: GPL Version 2, June 1991 see Linux/COPYING for details. + + The ``include-test`` (:py:class:`ParseTests`) directive calls produces + a documentation from KUnit/Kselftests based on the fields described + at the config variable below: + + - Feature: + Name of the feature to be tested + + - Mega feature: + Group of features that are under test + + - Scope: + Test scope. Usually, the name of the subsystem where the + driver belongs. + + - Description: + Description of the feature + + Overview of directive's argument and options. + + .. code-block:: rst + + .. include-test:: <C file> + :debug: + + The argument ``<C file>`` is required and should contain the location of + a KUnit or Kselftest C file with test documentation. + + ``debug`` + Inserts a code-block with the *raw* reST. Sometimes it is helpful to see + what reST is generated. + +""" + +from docutils import nodes +from docutils.statemachine import ViewList +from docutils.parsers.rst import Directive +from sphinx.util.docutils import switch_source_input + +from test_list import TestList + +__version__ = '1.0' + +# Valid fields for KUnit/Kselftest documentation +config = { + "fields": { + "Scope": { + "Mega feature": { + "Feature": { } + } + }, + "Description" : { } + } +} + +def setup(app): + u"""Setup ``include-test`` directive""" + + app.add_directive("include-test", ParseTests) + return dict( + version = __version__, + parallel_read_safe = True, + parallel_write_safe = True, + ) + +class ParseTests(Directive): + + u"""TestList parsing for ``include-test`` directive""" + + required_arguments = 1 + optional_arguments = 0 + has_content = False + final_argument_whitespace = True + + def run(self): + env = self.state.document.settings.env + doc = self.state.document + if not doc.settings.file_insertion_enabled: + raise self.warning("docutils: file insertion disabled") + + tests = TestList(config_dict = config, file_list = self.arguments, + test_tag = "KTEST_SUITE", subtest_tag = "KTEST_TEST", + main_name = "ktest") + + lines = tests.print_nested_rest(return_string = True) + + content = ViewList() + for line_no,line in enumerate(lines.split("\n")): + content.append(line, env.srcdir + "/" + env.docname, line_no) + + node = nodes.section() + with switch_source_input(self.state, content): + self.state.nested_parse(content, 0, node, match_titles=1) + + return node.children diff --git a/Documentation/sphinx/test_list.py b/Documentation/sphinx/test_list.py new file mode 100644 index 000000000000..402c69e93180 --- /dev/null +++ b/Documentation/sphinx/test_list.py @@ -0,0 +1,1288 @@ +#!/usr/bin/env python3 +# pylint: disable=C0301,R0902,R0914,R0912,R0913,R0915,R1702,C0302 +# SPDX-License-Identifier: (GPL-2.0 OR MIT) + +## Copyright (C) 2023 Intel Corporation ## +## Author: Mauro Carvalho Chehab mchehab@kernel.org ## +## ## +## Allow keeping inlined test documentation and validate ## +## if the documentation is kept updated. ## + +"""Maintain test plan and test implementation documentation on IGT.""" + +import glob +import json +import os +import re +import subprocess +import sys + +MIN_PYTHON = (3, 6) +if sys.version_info < MIN_PYTHON: + sys.exit("Python %s.%s or later is required.\n" % MIN_PYTHON) # pylint: disable=C0209 + +# +# ancillary functions to sort dictionary hierarchy +# +def _sort_per_level(item): + if "level" not in item[1]["_properties_"]: + return item[0] + + return "%05d_%05d_%s" % (item[1]["_properties_"]["level"], item[1]["_properties_"]["sublevel"], item[0]) # pylint: disable=C0209 + +def _sort_using_array(item, array): + ret_str = '' + for field in array: + if field in item[1]: + ret_str += '_' + field + '_' + item[1][field] + + if ret_str == '': + ret_str="________" + + return ret_str + +# +# Ancillary logic to allow plurals on fields +# +# As suggested at https://stackoverflow.com/questions/18902608/generating-the-plural-form-of-a... +# + +def _plural(field): + + """ + Poor man's conversion to plural. + + It should cover usual English plural rules, although it is not meant + to cover exceptions (except for a few ones that could be useful on actual + fields). + + """ + + match = re.match(r"(.*)\b(\S+)", field) + if match: + ret_str = match.group(1) + word = match.group(2) + + if word.isupper(): + ret_str += word + elif word in ["of", "off", "on", "description", "todo"]: + ret_str += word + elif word.endswith('ed'): + ret_str += word + elif word[-1:] in ['s', 'x', 'z']: + ret_str += word + 'es' + elif word[-2:] in ['sh', 'ch']: + ret_str += word + 'es' + elif word.endswith('fe'): + ret_str += word[:-2] + 'ves' + elif word.endswith('f'): + ret_str += word[:-1] + 'ves' + elif word.endswith('y'): + ret_str += word[:-1] + 'ies' + elif word.endswith('o'): + ret_str += word + 'es' + elif word.endswith('us'): + ret_str += word[:-2] + 'i' + elif word.endswith('on'): + ret_str += word[:-2] + 'a' + elif word.endswith('an'): + ret_str += word[:-2] + 'en' + else: + ret_str += word + 's' + + return ret_str + + return field + +# +# TestList class definition +# +class TestList: + + """ + Parse and handle test lists with test/subtest documentation, in the + form of C comments, with two meta-tags (by default, TEST and SUBTEST), + and a set of `field: value` items: + + /** + * TEST: Check if new IGT test documentation logic functionality is working + * Category: Software build block + * Sub-category: documentation + * Functionality: test documentation + * Issue: none + * Description: Complete description of this test + * + * SUBTEST: foo + * Description: do foo things + * description continuing on another line + * + * SUBTEST: bar + * Description: do bar things + * description continuing on another line + */ + + /** + * SUBTEST: test-%s-binds-%s-with-%ld-size + * Description: Test arg[2] arg[1] binds with arg[3] size + * + * SUBTEST: test-%s-%ld-size + * Description: Test arg[1] with %arg[2] size + * + * arg[1]: + * + * @large: large + * something + * @mixed: mixed + * something + * @small: small + * something + * + * arg[2]: buffer size + * of some type + * + * arg[3]: + * + * @binds: foo + * @misaligned-binds: misaligned + * @userptr-binds: userptr + * @userptr-misaligned-binds: userptr misaligned + */ + + It is also possible to define a list of values that will actually be + used by the test and no string replacement is needed. + This is done by using one or multiple arg[n].values lines: + + /** + * SUBTEST: unbind-all-%d-vmas + * Description: unbind all with %arg[1] VMAs + * + * arg[1].values: 2, 8 + * arg[1].values: 16, 32 + */ + + /** + * SUBTEST: unbind-all-%d-vmas + * Description: unbind all with %arg[1] VMAs + * + * arg[1].values: 64, 128 + */ + + Such feature is specially useful for numeric parameters + + The valid `field` values are defined on a JSON configuration file. + + The TEST documents the common fields present on all tests. Typically, + each IGT C test file should contain just one such field. + + The SUBTEST contains the fields that are specific to each subtest. + + Note: when igt_simple_main is used, there will be a single nameless + subtest. On such case, "SUBTEST:" is still needed, but without a + test name on it, e. g., it would be documented as: + + /** + * TEST: some test that uses igt_simple_main + * Category: Software build block + * Sub-category: documentation + * Functionality: test documentation + * Issue: none + * Description: Complete description of this test + * + * SUBTEST: + * Description: do foo things + */ + + Some IGT tests may have strings or integer wildcards, like: + test-%s-%ld-size + + Those are declared using igt_subtest_f() and similar macros. + + The wildcard arguments there need to be expanded. This is done by + defining arg[1] to arg[n] at the same code comment that contains the + SUBTEST as those variables are locally processed on each comment line. + + This script needs a configuration file, in JSON format, describing the + fields which will be parsed for TEST and/or SUBTEST tags. + + An example of such file is: + + { + "files": [ "tests/driver/*.c" ], + "exclude_files": [ "tests/driver/*-helper.c" ], + "fields": { + "Category": { + "Sub-category": { + "Functionality": { + } + } + }, + "Issue": { + "_properties_": { + "description": "If the test is used to solve an issue, point to the URL containing the issue." + } + }, + "Description" : { + "_properties_": { + "description": "Provides a description for the test/subtest." + } + } + } + } + + So, the above JSON config file expects tags like those: + + TEST: foo + Description: foo + + SUBTEST: bar + Category: Hardware + Sub-category: EU + Description: test bar on EU + + SUBTEST: foobar + Category: Software + Type: ioctl + Description: test ioctls + """ + + def __init__(self, config_fname = None, + include_plan = False, file_list = None, + built_testlist = None, + config_dict = None, sources_path = None, + test_tag = "TEST", subtest_tag = "SUBTESTS?", + main_name = "igt", subtest_separator = "@"): + self.doc = {} + self.test_number = 0 + self.config = None + self.filenames = file_list + self.plan_filenames = [] + self.props = {} + self.built_testlist = built_testlist + self.level_count = 0 + self.field_list = {} + self.title = None + self.filters = {} + self.subtest_separator = subtest_separator + self.main_name = main_name + + # Exclusive or: either one is needed + if bool(config_fname) == bool(config_dict): + sys.exit("Error: either config filename or config dict shall be used") + + if self.main_name: + self.main_name += subtest_separator + + implemented_class = None + + if config_fname: + with open(config_fname, 'r', encoding='utf8') as handle: + self.config = json.load(handle) + + config_origin = config_fname + cfg_path = os.path.realpath(os.path.dirname(config_fname)) + "/" + driver_name = re.sub(r'(.*/)?([^/]+)/.*', r'\2', config_fname).capitalize() + + else: + self.config = config_dict + config_origin = "config dict" + cfg_path = "./" + driver_name = main_name + + if sources_path: + cfg_path = os.path.realpath(sources_path) + "/" + + if not self.config: + sys.exit("Error: configuration is empty!") + + self.__add_field(None, 0, 0, self.config["fields"]) + + sublevel_count = [ 0 ] * self.level_count + + for field, item in self.props.items(): + if "sublevel" in item["_properties_"]: + level = item["_properties_"]["level"] + sublevel = item["_properties_"]["sublevel"] + if sublevel > sublevel_count[level - 1]: + sublevel_count[level - 1] = sublevel + + field_lc = field.lower() + self.field_list[field_lc] = field + field_plural = _plural(field_lc) + if field_lc != field_plural: + self.field_list[field_plural] = field + + if include_plan: + self.props["Class"] = {} + self.props["Class"]["_properties_"] = {} + self.props["Class"]["_properties_"]["level"] = 1 + self.props["Class"]["_properties_"]["sublevel"] = sublevel_count[0] + 1 + + # Remove non-multilevel items, as we're only interested on + # hierarchical item levels here + for field, item in self.props.items(): + if "sublevel" in item["_properties_"]: + level = item["_properties_"]["level"] + if sublevel_count[level - 1] == 1: + del item["_properties_"]["level"] + del item["_properties_"]["sublevel"] + if "_properties_" in self.props: + del self.props["_properties_"] + + has_implemented = False + if not self.filenames: + self.filenames = [] + exclude_files = [] + files = self.config["files"] + exclude_file_glob = self.config.get("exclude_files", []) + for cfg_file in exclude_file_glob: + cfg_file = cfg_path + cfg_file + for fname in glob.glob(cfg_file): + exclude_files.append(fname) + + for cfg_file in files: + cfg_file = cfg_path + cfg_file + for fname in glob.glob(cfg_file): + if fname in exclude_files: + continue + self.filenames.append(fname) + has_implemented = True + else: + for cfg_file in self.filenames: + if cfg_file: + has_implemented = True + + has_planned = False + if include_plan and "planning_files" in self.config: + implemented_class = "Implemented" + files = self.config["planning_files"] + for cfg_file in files: + cfg_file = cfg_path + cfg_file + for fname in glob.glob(cfg_file): + self.plan_filenames.append(fname) + has_planned = True + + planned_class = None + if has_implemented: + if has_planned: + planned_class = "Planned" + self.title = "Planned and implemented " + else: + self.title = "Implemented " + else: + if has_planned: + self.title = "Planned " + else: + sys.exit("Need file names to be processed") + + self.title += self.config.get("name", "tests for " + driver_name + " driver") + + # Parse files, expanding wildcards + field_re = re.compile(r"(" + '|'.join(self.field_list.keys()) + r'):\s*(.*)', re.I) + + for fname in self.filenames: + if fname == '': + continue + + self.__add_file_documentation(fname, implemented_class, field_re, + test_tag, subtest_tag, config_origin) + + if include_plan: + for fname in self.plan_filenames: + self.__add_file_documentation(fname, planned_class, field_re, + test_tag, subtest_tag, config_origin) + + # + # ancillary methods + # + + def __add_field(self, name, sublevel, hierarchy_level, field): + + """ Flatten config fields into a non-hierarchical dictionary """ + + for key in field: + if key not in self.props: + self.props[key] = {} + self.props[key]["_properties_"] = {} + + if name: + if key == "_properties_": + if key not in self.props: + self.props[key] = {} + self.props[name][key].update(field[key]) + + sublevel += 1 + hierarchy_level += 1 + if "sublevel" in self.props[name][key]: + if self.props[name][key]["sublevel"] != sublevel: + sys.exit(f"Error: config defined {name} as sublevel {self.props[key]['sublevel']}, but wants to redefine as sublevel {sublevel}") + + self.props[name][key]["level"] = self.level_count + self.props[name][key]["sublevel"] = sublevel + + continue + else: + self.level_count += 1 + + self.__add_field(key, sublevel, hierarchy_level, field[key]) + + def __filter_subtest(self, test, subtest, field_not_found_value): + + """ Apply filter criteria to subtests """ + + for filter_field, regex in self.filters.items(): + if filter_field in subtest: + if not regex.match(subtest[filter_field]): + return True + elif filter_field in test: + if not regex.match(test[filter_field]): + return True + else: + return field_not_found_value + + # None of the filtering rules were applied + return False + + def expand_subtest(self, fname, test_name, test, allow_inherit, with_lines = False, with_subtest_nr = False): + + """Expand subtest wildcards providing an array with subtests""" + + subtest_array = [] + + for subtest in self.doc[test]["subtest"].keys(): + summary = test_name + if self.doc[test]["subtest"][subtest]["_summary_"] != '': + summary += self.subtest_separator + self.doc[test]["subtest"][subtest]["_summary_"] + if not summary: + continue + + num_vars = summary.count('%') + file_ln = self.doc[test]["subtest_line"][subtest] + + # Handle trivial case: no wildcards + if num_vars == 0: + subtest_dict = {} + + subtest_dict["_summary_"] = summary + + for k in sorted(self.doc[test]["subtest"][subtest].keys()): + if k in [ '_summary_', 'arg', 'subtest_line' ]: + continue + + if not allow_inherit: + if k in self.doc[test] and self.doc[test]["subtest"][subtest][k] == self.doc[test][k]: + continue + + subtest_dict[k] = self.doc[test]["subtest"][subtest][k] + + if with_lines: + subtest_dict["line"] = file_ln + + if with_subtest_nr: + subtest_dict["subtest_nr"] = subtest + + subtest_array.append(subtest_dict) + + continue + + # Handle summaries with wildcards + + # Convert subtest arguments into an array + arg_array = {} + arg_ref = self.doc[test]["subtest"][subtest]["arg"] + + for arg_k in self.doc[test]["arg"][arg_ref].keys(): + arg_array[arg_k] = [] + if int(arg_k) > num_vars: + continue + + for arg_el in sorted(self.doc[test]["arg"][arg_ref][arg_k].keys()): + arg_array[arg_k].append(arg_el) + + size = len(arg_array) + + if size < num_vars: + sys.exit(f"{fname}:subtest {summary} needs {num_vars} arguments but only {size} are defined.") + + for j in range(0, num_vars): + if arg_array[j] is None: + sys.exit(f"{fname}:subtest{summary} needs arg[{j}], but this is not defined.") + + + # convert numeric wildcards to string ones + summary = re.sub(r'%(d|ld|lld|i|u|lu|llu)','%s', summary) + + # Expand subtests + pos = [ 0 ] * num_vars + args = [ 0 ] * num_vars + arg_map = [ 0 ] * num_vars + while 1: + for j in range(0, num_vars): + arg_val = arg_array[j][pos[j]] + args[j] = arg_val + + if arg_val in self.doc[test]["arg"][arg_ref][j]: + arg_map[j] = self.doc[test]["arg"][arg_ref][j][arg_val] + if re.match(r"<.*>", self.doc[test]["arg"][arg_ref][j][arg_val]): + args[j] = "<" + arg_val + ">" + else: + arg_map[j] = arg_val + + arg_summary = summary % tuple(args) + + # Store the element + subtest_dict = {} + subtest_dict["_summary_"] = arg_summary + + for field in sorted(self.doc[test]["subtest"][subtest].keys()): + if field in [ '_summary_', 'arg', 'subtest_line' ]: + continue + + sub_field = self.doc[test]["subtest"][subtest][field] + sub_field = re.sub(r"%?\barg[(\d+)]", lambda m: arg_map[int(m.group(1)) - 1], sub_field) # pylint: disable=W0640 + + if not allow_inherit: + if field in self.doc[test]: + if sub_field in self.doc[test][field] and sub_field == self.doc[test][field]: + continue + + subtest_dict[field] = sub_field + + if with_lines: + subtest_dict["line"] = file_ln + + if with_subtest_nr: + subtest_dict["subtest_nr"] = subtest + + subtest_array.append(subtest_dict) + + # Increment variable inside the array + arr_pos = 0 + while pos[arr_pos] + 1 >= len(arg_array[arr_pos]): + pos[arr_pos] = 0 + arr_pos += 1 + if arr_pos >= num_vars: + break + + if arr_pos >= num_vars: + break + + pos[arr_pos] += 1 + + return subtest_array + + def expand_dictionary(self, subtest_only): + + """ prepares a dictionary with subtest arguments expanded """ + + test_dict = {} + + for test in self.doc: # pylint: disable=C0206 + fname = self.doc[test]["File"] + + name = re.sub(r'.*/', '', fname) + name = re.sub(r'.[\w+]$', '', name) + name = self.main_name + name + + if not subtest_only: + test_dict[name] = {} + + for field in self.doc[test]: + if field == "subtest": + continue + if field == "arg": + continue + + test_dict[name][field] = self.doc[test][field] + dic = test_dict[name] + else: + dic = test_dict + + subtest_array = self.expand_subtest(fname, name, test, subtest_only) + for subtest in subtest_array: + if self.__filter_subtest(self.doc[test], subtest, True): + continue + + summary = subtest["_summary_"] + + dic[summary] = {} + for field in sorted(subtest.keys()): + if field in [ '_summary_', 'arg', 'subtest_line' ]: + continue + dic[summary][field] = subtest[field] + + return test_dict + + # + # Output methods + # + + def print_rest_flat(self, filename = None, return_string = False): + + """Print tests and subtests ordered by tests""" + + out = "=" * len(self.title) + "\n" + out += self.title + "\n" + out += "=" * len(self.title) + "\n\n" + + for test in sorted(self.doc.keys()): + fname = self.doc[test]["File"] + + name = re.sub(r'.*/', '', fname) + name = re.sub(r'.[ch]', '', name) + name = self.main_name + name + + tmp_subtest = self.expand_subtest(fname, name, test, False) + + # Get subtests first, to avoid displaying tests with + # all filtered subtests + subtest_array = [] + for subtest in tmp_subtest: + if self.__filter_subtest(self.doc[test], subtest, True): + continue + subtest_array.append(subtest) + + if not subtest_array: + continue + + out += len(name) * '=' + "\n" + out += name + "\n" + out += len(name) * '=' + "\n\n" + + for field in sorted(self.doc[test].keys()): + if field == "subtest": + continue + if field == "arg": + continue + if field == "_summary_": + continue + if field == "subtest_line": + continue + + out += f":{field}: {self.doc[test][field]}\n" + + for subtest in subtest_array: + + out += "\n" + subtest["_summary_"] + "\n" + out += len(subtest["_summary_"]) * '=' + "\n\n" + + for field in sorted(subtest.keys()): + if field in [ '_summary_', 'arg', 'subtest_line' ]: + continue + + out += f":{field}:" + subtest[field] + "\n" + + out += "\n" + + out += "\n\n" + + if filename: + with open(filename, "w", encoding='utf8') as handle: + handle.write(out) + elif not return_string: + print(out) + else: + return out + + def get_spreadsheet(self): + + """ + Return a bidimentional array with the test contents. + + Its output is similar to a spreadsheet, so it can be used by a + separate python file that would create a workbook's sheet. + """ + + sheet = [] + row = 0 + sheet.append([]) + sheet[row].append('Test name') + + subtest_dict = self.expand_dictionary(True) + + # Identify the sort order for the fields + fields_order = [] + fields = sorted(self.props.items(), key = _sort_per_level) + for item in fields: + fields_order.append(item[0]) + sheet[row].append(item[0]) + + # Receives a flat subtest dictionary, with wildcards expanded + subtest_dict = self.expand_dictionary(True) + + subtests = sorted(subtest_dict.items(), + key = lambda x: _sort_using_array(x, fields_order)) + + for subtest, fields in subtests: + row += 1 + sheet.append([]) + + sheet[row].append(subtest) + + for field in fields_order: + if field in fields: + sheet[row].append(fields[field]) + else: + sheet[row].append('') + return sheet + + def print_nested_rest(self, filename = None, return_string = False): + + """Print tests and subtests ordered by tests""" + + handler = None + if filename: + original_stdout = sys.stdout + handler = open(filename, "w", encoding='utf8') # pylint: disable=R1732 + sys.stdout = handler + + out = "=" * len(self.title) + "\n" + out += self.title + "\n" + out += "=" * len(self.title) + "\n\n" + + # Identify the sort order for the fields + fields_order = [] + fields = sorted(self.props.items(), key = _sort_per_level) + for item in fields: + fields_order.append(item[0]) + + # Receives a flat subtest dictionary, with wildcards expanded + subtest_dict = self.expand_dictionary(True) + + subtests = sorted(subtest_dict.items(), + key = lambda x: _sort_using_array(x, fields_order)) + + # Use the level markers below + level_markers='=-^_~:.`"*+#' + + # Print the data + old_fields = [ '' ] * len(fields_order) + + for subtest, fields in subtests: + # Check what level has different message + marker = 0 + for cur_level in range(0, len(fields_order)): # pylint: disable=C0200 + field = fields_order[cur_level] + if "level" not in self.props[field]["_properties_"]: + continue + if field in fields: + if old_fields[cur_level] != fields[field]: + break + marker += 1 + + # print hierarchy + for i in range(cur_level, len(fields_order)): + if "level" not in self.props[fields_order[i]]["_properties_"]: + continue + if fields_order[i] not in fields: + continue + + if marker >= len(level_markers): + sys.exit(f"Too many levels: {marker}, maximum limit is {len(level_markers):}") + + title_str = fields_order[i] + ": " + fields[fields_order[i]] + + out += title_str + "\n" + out += level_markers[marker] * len(title_str) + "\n\n" + marker += 1 + + out += "\n``" + subtest + "``" + "\n\n" + + # print non-hierarchy fields + for field in fields_order: + if "level" in self.props[field]["_properties_"]: + continue + + if field in fields: + out += f":{field}: {fields[field]}" + "\n" + + # Store current values + for i in range(cur_level, len(fields_order)): + field = fields_order[i] + if "level" not in self.props[field]["_properties_"]: + continue + if field in fields: + old_fields[i] = fields[field] + else: + old_fields[i] = '' + + out += "\n" + + if filename: + with open(filename, "w", encoding='utf8') as handle: + handle.write(out) + elif not return_string: + print(out) + else: + return out + + def print_json(self, out_fname): + + """Adds the contents of test/subtest documentation form a file""" + + # Receives a dictionary with tests->subtests with expanded subtests + test_dict = self.expand_dictionary(False) + + with open(out_fname, "w", encoding='utf8') as write_file: + json.dump(test_dict, write_file, indent = 4) + + # + # Add filters + # + def add_filter(self, filter_field_expr): + + """ Add a filter criteria for output data """ + + match = re.match(r"(.*)=~\s*(.*)", filter_field_expr) + if not match: + sys.exit(f"Filter field {filter_field_expr} is not at <field> =~ <regex> syntax") + field = match.group(1).strip().lower() + if field not in self.field_list: + sys.exit(f"Field '{field}' is not defined") + filter_field = self.field_list[field] + regex = re.compile("{0}".format(match.group(2).strip()), re.I) # pylint: disable=C0209 + self.filters[filter_field] = regex + + # + # Subtest list methods + # + + def get_subtests(self, sort_field = None, expand = None, with_order = False): + + """Return an array with all subtests""" + + subtests = {} + subtests[""] = [] + + order = None + + if sort_field: + if sort_field.lower() not in self.field_list: + sys.exit(f"Field '{sort_field}' is not defined") + sort_field = self.field_list[sort_field.lower()] + + if with_order: + if "_properties_" in self.props[sort_field]: + if "order" in self.props[sort_field]["_properties_"]: + order = self.props[sort_field]["_properties_"]["order"] + + subtest_array = [] + for test in sorted(self.doc.keys()): + fname = self.doc[test]["File"] + + test_name = re.sub(r'.*/', '', fname) + test_name = re.sub(r'.[ch]\s*', '', test_name) + test_name = self.main_name + test_name + + subtest_array += self.expand_subtest(fname, test_name, test, True) + + subtest_array.sort(key = lambda x : x.get('_summary_')) + + for subtest in subtest_array: + if self.__filter_subtest(self.doc[test], subtest, True): + continue + + if sort_field: + if sort_field in subtest: + if expand: + test_list = subtest[sort_field].split(expand) + test_list = [s.strip() for s in test_list] + + for test_elem in test_list: + if test_elem not in subtests: + subtests[test_elem] = [] + if order: + subtests[test_elem].append((subtest["_summary_"], test_list)) + else: + subtests[test_elem].append(subtest["_summary_"]) + else: + if subtest[sort_field] not in subtests: + subtests[subtest[sort_field]] = [] + if order: + subtests[test_elem].append((subtest["_summary_"], [subtest[sort_field]])) + else: + subtests[subtest[sort_field]].append(subtest["_summary_"]) + else: + if order: + subtests[test_elem].append((subtest["_summary_"], [subtest[sort_field]])) + else: + subtests[""].append(subtest["_summary_"]) + + else: + if order: + subtests[test_elem].append((subtest["_summary_"], [subtest[sort_field]])) + else: + subtests[""].append(subtest["_summary_"]) + + if order: + for group, tests in subtests.items(): + prefix_tests = [] + suffix_tests = [] + middle_tests = [] + is_prefix = True + for k in order: + if k == "__all__": + is_prefix = False + continue + for test in tests: + if k in test[1]: + if is_prefix: + prefix_tests.append(test[0]) + else: + suffix_tests.append(test[0]) + for test in tests: + if test[0] not in prefix_tests and test[0] not in suffix_tests: + middle_tests.append(test[0]) + subtests[group] = prefix_tests + middle_tests + suffix_tests + + return subtests + + # + # Validation methods + # + def check_tests(self): + + """Compare documented subtests with the test list""" + + if not self.built_testlist: + sys.exit("Need the build path where the exec files are located") + + if self.filters: + print("NOTE: test checks are affected by filters") + + doc_subtests = set() + + args_regex = re.compile(r'<[^>]+>') + + for subtest in self.get_subtests()[""]: + subtest = self.subtest_separator.join(subtest.split(self.subtest_separator)[:3]) + subtest = re.sub(r'<[^>]+>', r'\d+', subtest) + doc_subtests.add(subtest) + + doc_subtests = list(sorted(doc_subtests)) + + # Get a list of tests from + tests = set() + for name in self.filenames: + test = self.main_name + re.sub(r".c$", "", name.split('/')[-1]) + tests.add(test) + + run_subtests = [] + with open(self.built_testlist, 'r', encoding='utf8') as handle: + for line in handle: + name = line.rstrip("\n") + if name in tests: + run_subtests.append(name) + else: + result = name.rsplit(self.subtest_separator, 1)[0] + if name in tests: + run_subtests.append(name) + + # Compare arrays + + run_missing = [] + doc_uneeded = [] + + test_regex = r"" + for doc_test in doc_subtests: + if test_regex != r"": + test_regex += r"|" + test_regex += doc_test + + test_regex = re.compile(r'^(' + test_regex + r')$') + + for doc_test in doc_subtests: + found = False + for run_test in run_subtests: + if test_regex.match(run_test): + found = True + break + if not found: + doc_uneeded.append(doc_test) + + for run_test in run_subtests: + found = False + if test_regex.match(run_test): + found = True + if not found: + run_missing.append(run_test) + + if doc_uneeded: + for test_name in doc_uneeded: + print(f"Warning: Documented {test_name} doesn't exist on source files") + + if run_missing: + for test_name in run_missing: + print(f'Warning: Missing documentation for {test_name}') + if doc_uneeded or run_missing: + sys.exit(1) + + # + # File handling methods + # + + def __add_file_documentation(self, fname, implemented_class, field_re, + test_tag, subtest_tag, config_origin): + + """Adds the contents of test/subtest documentation form a file""" + + current_test = None + current_subtest = None + + handle_section = '' + current_field = '' + arg_number = 1 + cur_arg = -1 + cur_arg_element = 0 + has_test_or_subtest = 0 + + test_regex = re.compile(test_tag + r':\s*(.*)') + subtest_regex = re.compile('^' + subtest_tag + r':\s*(.*)') + + with open(fname, 'r', encoding='utf8') as handle: + arg_ref = None + current_test = '' + subtest_number = 0 + + for file_ln, file_line in enumerate(handle): + file_line = file_line.rstrip() + + if re.match(r'^\s**$', file_line): + continue + + if re.match(r'^\s**/$', file_line): + handle_section = '' + current_subtest = None + arg_ref = None + cur_arg = -1 + has_test_or_subtest = 0 + + continue + + if re.match(r'^\s*/**$', file_line): + handle_section = '1' + continue + + if not handle_section: + continue + + file_line = re.sub(r'^\s** ?', '', file_line) + + # Handle only known sections + if handle_section == '1': + current_field = '' + + # Check if it is a new TEST section + match = test_regex.match(file_line) + if match: + has_test_or_subtest = 1 + current_test = self.test_number + self.test_number += 1 + + handle_section = 'test' + + self.doc[current_test] = {} + self.doc[current_test]["arg"] = {} + self.doc[current_test]["_summary_"] = match.group(1) + self.doc[current_test]["File"] = fname + self.doc[current_test]["subtest"] = {} + self.doc[current_test]["subtest_line"] = {} + + if implemented_class: + self.doc[current_test]["Class"] = implemented_class + current_subtest = None + + continue + + # Check if it is a new SUBTEST section + match = subtest_regex.match(file_line) + if match: + has_test_or_subtest = 1 + current_subtest = subtest_number + subtest_number += 1 + + current_field = '' + handle_section = 'subtest' + + # subtests inherit properties from the tests + self.doc[current_test]["subtest"][current_subtest] = {} + for field in self.doc[current_test].keys(): + if field == "arg": + continue + if field == "summary": + continue + if field == "File": + continue + if field == "subtest": + continue + if field == "_properties_": + continue + if field == "Description": + continue + self.doc[current_test]["subtest"][current_subtest][field] = self.doc[current_test][field] + + self.doc[current_test]["subtest"][current_subtest]["_summary_"] = match.group(1) + self.doc[current_test]["subtest"][current_subtest]["Description"] = '' + self.doc[current_test]["subtest_line"][current_subtest] = file_ln + + if not arg_ref: + arg_ref = arg_number + arg_number += 1 + self.doc[current_test]["arg"][arg_ref] = {} + + self.doc[current_test]["subtest"][current_subtest]["arg"] = arg_ref + + continue + + # It is a known section. Parse its contents + match = field_re.match(file_line) + if match: + current_field = self.field_list[match.group(1).lower()] + match_val = match.group(2) + + if handle_section == 'test': + self.doc[current_test][current_field] = match_val + else: + self.doc[current_test]["subtest"][current_subtest][current_field] = match_val + + cur_arg = -1 + + continue + + if not has_test_or_subtest: + continue + + # Use hashes for arguments to avoid duplication + match = re.match(r'arg[(\d+)]:\s*(.*)', file_line) + if match: + current_field = '' + if arg_ref is None: + sys.exit(f"{fname}:{file_ln + 1}: arguments should be defined after one or more subtests, at the same comment") + + cur_arg = int(match.group(1)) - 1 + if cur_arg not in self.doc[current_test]["arg"][arg_ref]: + self.doc[current_test]["arg"][arg_ref][cur_arg] = {} + + cur_arg_element = match.group(2) + + if match.group(2): + # Should be used only for numeric values + self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] = "<" + match.group(2) + ">" + + continue + + match = re.match(r'@(\S+):\s*(.*)', file_line) + if match: + if cur_arg >= 0: + current_field = '' + if arg_ref is None: + sys.exit(f"{fname}:{file_ln + 1}: arguments should be defined after one or more subtests, at the same comment") + + cur_arg_element = match.group(1) + self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] = match.group(2) + + else: + print(f"{fname}:{file_ln + 1}: Warning: invalid argument: @%s: %s" % + (match.group(1), match.group(2))) + + continue + + # We don't need a multi-lined version here + match = re.match(r'arg[(\d+)].values:\s*(.*)', file_line) + if match: + cur_arg = int(match.group(1)) - 1 + + if cur_arg not in self.doc[current_test]["arg"][arg_ref]: + self.doc[current_test]["arg"][arg_ref][cur_arg] = {} + + values = match.group(2).replace(" ", "").split(",") + for split_val in values: + if split_val == "": + continue + self.doc[current_test]["arg"][arg_ref][cur_arg][split_val] = split_val + + continue + + # Handle multi-line field contents + if current_field: + match = re.match(r'\s+(.*)', file_line) + if match: + if handle_section == 'test': + dic = self.doc[current_test] + else: + dic = self.doc[current_test]["subtest"][current_subtest] + + if dic[current_field] != '': + dic[current_field] += " " + dic[current_field] += match.group(1) + continue + + # Handle multi-line argument contents + if cur_arg >= 0 and arg_ref is not None: + match = re.match(r'\s+*?\s*(.*)', file_line) + if match: + match_val = match.group(1) + + match = re.match(r'^(<.*)>$',self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element]) + if match: + self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] = match.group(1) + ' ' + match_val + ">" + else: + if self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] != '': + self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] += ' ' + self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] += match_val + continue + + file_line.rstrip(r"\n") + sys.exit(f"{fname}:{file_ln + 1}: Error: unrecognized line. Need to add field at %s?\n\t==> %s" % + (config_origin, file_line)) + + def show_subtests(self, sort_field): + + """Show subtests, allowing sort and filter a field """ + + if sort_field: + test_subtests = self.get_subtests(sort_field) + for val_key in sorted(test_subtests.keys()): + if not test_subtests[val_key]: + continue + if val_key == "": + print("not defined:") + else: + print(f"{val_key}:") + for sub in test_subtests[val_key]: + print (f" {sub}") + else: + for sub in self.get_subtests(sort_field)[""]: + print (sub) + + def gen_testlist(self, directory, sort_field): + + """Generate testlists from the test documentation""" + + test_prefix = os.path.commonprefix(self.get_subtests()[""]) + test_prefix = re.sub(r'^' + self.main_name, '', test_prefix) + + # NOTE: currently, it uses a comma for multi-value delimitter + test_subtests = self.get_subtests(sort_field, ",", with_order = True) + + if not os.path.exists(directory): + os.makedirs(directory) + + for test in test_subtests.keys(): # pylint: disable=C0201,C0206 + if not test_subtests[test]: + continue + + testlist = test.lower() + if testlist == "": + fname = "other" + elif testlist == "bat": # Basic Acceptance Test + fname = "fast-feedback" + else: + fname = testlist + + fname = re.sub(r"[^\w\d]+", "-", fname) + fname = directory + "/" + test_prefix + fname + ".testlist" + + with open(fname, 'w', encoding='utf8') as handler: + for sub in test_subtests[test]: + handler.write (f"{sub}\n") + print(f"{fname} created.")