[PATCH v2 1/4] kunit: tool: Do not error on tests without test plans

List overview All Threads
Download

newer

older

[PATCH] selftest/sgx: use swap()...

[PATCH v2 0/3] Small userfaultfd...

David Gow

27 Oct 2021 27 Oct '21

1:36 a.m.

The (K)TAP spec encourages test output to begin with a 'test plan': a count of the number of tests being run of the form: 1..n

However, some test suites might not know the number of subtests in advance (for example, KUnit's parameterised tests use a generator function). In this case, it's not possible to print the test plan in advance.

kunit_tool already parses test output which doesn't contain a plan, but reports an error. Since we want to use nested subtests with KUnit paramterised tests, remove this error.

Signed-off-by: David Gow davidgow@google.com --- tools/testing/kunit/kunit_parser.py | 5 ++--- tools/testing/kunit/kunit_tool_test.py | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 3355196d0515..50ded55c168c 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -340,8 +340,8 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: """ Parses test plan line and stores the expected number of subtests in test object. Reports an error if expected count is 0. - Returns False and reports missing test plan error if fails to parse - test plan. + Returns False and sets expected_count to None if there is no valid test + plan.

Accepted format: - '1..[number of subtests]' @@ -356,7 +356,6 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: match = TEST_PLAN.match(lines.peek()) if not match: test.expected_count = None - test.add_error('missing plan line!') return False test.log.append(lines.pop()) expected_count = int(match.group(1)) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 9c4126731457..bc8793145713 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -191,7 +191,10 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines( file.readlines())) - self.assertEqual(2, result.test.counts.errors) + # A missing test plan is not an error. + self.assertEqual(0, result.test.counts.errors) + # All tests should be accounted for. + self.assertEqual(10, result.test.counts.total()) self.assertEqual( kunit_parser.TestStatus.SUCCESS, result.status)

-- 2.33.0.1079.g6e70778dc9-goog

Show replies by date

David Gow

27 Oct 27 Oct

1:37 a.m.

New subject: [PATCH v2 2/4] kunit: tool: Report an error if any test has no subtests

It's possible for a test to have a subtest header, but zero valid subtests. We used to error on this if the test plan had no subtests listed, but it's possible to have subtests without a test plan (indeed, this is how parameterised tests work).

Tests with 0 subtests now have the result NO_TESTS, and will report an error (which does not halt test execution, but is printed in a scary red colour and is noted in the results summary).

Signed-off-by: David Gow davidgow@google.com --- tools/testing/kunit/kunit_parser.py | 14 +++++++++----- tools/testing/kunit/kunit_tool_test.py | 9 +++++++++ .../test_is_test_passed-no_tests_no_plan.log | 7 +++++++ 3 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 tools/testing/kunit/test_data/test_is_test_passed-no_tests_no_plan.log

diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 50ded55c168c..3a838423c381 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -360,9 +360,6 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: test.log.append(lines.pop()) expected_count = int(match.group(1)) test.expected_count = expected_count - if expected_count == 0: - test.status = TestStatus.NO_TESTS - test.add_error('0 tests run!') return True

TEST_RESULT = re.compile(r'^(ok|not ok) ([0-9]+) (- )?([^#]*)( # .*)?$') @@ -731,6 +728,7 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str]) -> Test: # test plan test.name = "main" parse_test_plan(lines, test) + parent_test = True else: # If KTAP/TAP header is not found, test must be subtest # header or test result line so parse attempt to parser @@ -744,7 +742,7 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str]) -> Test: expected_count = test.expected_count subtests = [] test_num = 1 - while expected_count is None or test_num <= expected_count: + while parent_test and (expected_count is None or test_num <= expected_count): # Loop to parse any subtests. # Break after parsing expected number of tests or # if expected number of tests is unknown break when test @@ -779,9 +777,15 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str]) -> Test: parse_test_result(lines, test, expected_num) else: test.add_error('missing subtest result line!') + + # Check for there being no tests + if parent_test and len(subtests) == 0: + test.status = TestStatus.NO_TESTS + test.add_error('0 tests run!') + # Add statuses to TestCounts attribute in Test object bubble_up_test_results(test) - if parent_test: + if parent_test and not main: # If test has subtests and is not the main test object, print # footer. print_test_footer(test) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index bc8793145713..c59fe0777387 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -208,6 +208,15 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual( kunit_parser.TestStatus.NO_TESTS, result.status) + no_plan_log = test_data_path('test_is_test_passed-no_tests_no_plan.log') + with open(no_plan_log) as file: + result = kunit_parser.parse_run_tests( + kunit_parser.extract_tap_lines(file.readlines())) + self.assertEqual(0, len(result.test.subtests[0].subtests[0].subtests)) + self.assertEqual( + kunit_parser.TestStatus.NO_TESTS, + result.test.subtests[0].subtests[0].status) +

def test_no_kunit_output(self): crash_log = test_data_path('test_insufficient_memory.log') diff --git a/tools/testing/kunit/test_data/test_is_test_passed-no_tests_no_plan.log b/tools/testing/kunit/test_data/test_is_test_passed-no_tests_no_plan.log new file mode 100644 index 000000000000..dd873c981108 --- /dev/null +++ b/tools/testing/kunit/test_data/test_is_test_passed-no_tests_no_plan.log @@ -0,0 +1,7 @@ +TAP version 14 +1..1 + # Subtest: suite + 1..1 + # Subtest: case + ok 1 - case # SKIP +ok 1 - suite

-- 2.33.0.1079.g6e70778dc9-goog

David Gow

1:37 a.m.

New subject: [PATCH v2 3/4] kunit: Don't crash if no parameters are generated

It's possible that a parameterised test could end up with zero parameters. At the moment, the test function will nevertheless be called with NULL as the parameter. Instead, don't try to run the test code, and just mark the test as SKIPped.

Reported-by: Daniel Latypov dlatypov@google.com Signed-off-by: David Gow davidgow@google.com --- lib/kunit/test.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 3bd741e50a2d..e028d98e4f5b 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -500,7 +500,10 @@ int kunit_run_tests(struct kunit_suite *suite) kunit_print_subtest_start(suite);

kunit_suite_for_each_test_case(suite, test_case) { - struct kunit test = { .param_value = NULL, .param_index = 0 }; + /* The initial param value is nonzero, as we want + * non-parametrised tests to run once. + */ + struct kunit test = { .param_value = (void *)-1, .param_index = 0 }; struct kunit_result_stats param_stats = { 0 }; test_case->status = KUNIT_SKIPPED;

@@ -510,7 +513,7 @@ int kunit_run_tests(struct kunit_suite *suite) test.param_value = test_case->generate_params(NULL, param_desc); }

- do { + while (test.param_value) { kunit_run_case_catch_errors(suite, test_case, &test);

if (test_case->generate_params) { @@ -530,11 +533,12 @@ int kunit_run_tests(struct kunit_suite *suite) param_desc[0] = '\0'; test.param_value = test_case->generate_params(test.param_value, param_desc); test.param_index++; - } + } else + test.param_value = NULL;

kunit_update_stats(&param_stats, test.status);

- } while (test.param_value); + }

kunit_print_test_stats(&test, param_stats);

-- 2.33.0.1079.g6e70778dc9-goog

Daniel Latypov

6:59 p.m.

New subject: [PATCH v2 3/4] kunit: Don't crash if no parameters are generated

On Tue, Oct 26, 2021 at 6:37 PM David Gow davidgow@google.com wrote:

...

It's possible that a parameterised test could end up with zero parameters. At the moment, the test function will nevertheless be called with NULL as the parameter. Instead, don't try to run the test code, and just mark the test as SKIPped.

Reported-by: Daniel Latypov dlatypov@google.com Signed-off-by: David Gow davidgow@google.com

lib/kunit/test.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 3bd741e50a2d..e028d98e4f5b 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -500,7 +500,10 @@ int kunit_run_tests(struct kunit_suite *suite) kunit_print_subtest_start(suite);
    kunit_suite_for_each_test_case(suite, test_case) {
          struct kunit test = { .param_value = NULL, .param_index = 0 };
          /* The initial param value is nonzero, as we want
           * non-parametrised tests to run once.
           */
          struct kunit test = { .param_value = (void *)-1, .param_index = 0 };

(Not a strong preference)

Hmm, I'd slightly prefer we don't set a dummy value of -1 for this. I personally think something like this is a bit less subtle:

/* Run non-parameterised tests once */ while (!test_case->generate_param || test.param_value) {

if (!test_case->generate_param) break; }

Alternatively, we don't need to share the loop

if (!test_case->generate_param) { kunit_run_case_catch_errors(suite, test_case, &test); kunit_update_stats(&param_stats, test.status); } else while (test_param.value) { kunit_run_case_catch_errors(suite, test_case, &test); kunit_update_stats(&param_stats, test.status); /* print subtest and advance next param */ }

}

...

            struct kunit_result_stats param_stats = { 0 };
            test_case->status = KUNIT_SKIPPED;
@@ -510,7 +513,7 @@ int kunit_run_tests(struct kunit_suite *suite) test.param_value = test_case->generate_params(NULL, param_desc); }
          do {
          while (test.param_value) {
                  kunit_run_case_catch_errors(suite, test_case, &test);

                  if (test_case->generate_params) {
@@ -530,11 +533,12 @@ int kunit_run_tests(struct kunit_suite *suite) param_desc[0] = '\0'; test.param_value = test_case->generate_params(test.param_value, param_desc); test.param_index++;
                  }
                  } else
                          test.param_value = NULL;

                  kunit_update_stats(&param_stats, test.status);
          } while (test.param_value);
          }

          kunit_print_test_stats(&test, param_stats);
-- 2.33.0.1079.g6e70778dc9-goog

David Gow

1:37 a.m.

New subject: [PATCH v2 4/4] kunit: Report test parameter results as (K)TAP subtests

Currently, the results for individial parameters in a parameterised test are simply output as (K)TAP diagnostic lines.

As kunit_tool now supports nested subtests, report each parameter as its own subtest.

For example, here's what the output now looks like: # Subtest: inode_test_xtimestamp_decoding ok 1 - 1901-12-13 Lower bound of 32bit < 0 timestamp, no extra bits ok 2 - 1969-12-31 Upper bound of 32bit < 0 timestamp, no extra bits ok 3 - 1970-01-01 Lower bound of 32bit >=0 timestamp, no extra bits ok 4 - 2038-01-19 Upper bound of 32bit >=0 timestamp, no extra bits ok 5 - 2038-01-19 Lower bound of 32bit <0 timestamp, lo extra sec bit on ok 6 - 2106-02-07 Upper bound of 32bit <0 timestamp, lo extra sec bit on ok 7 - 2106-02-07 Lower bound of 32bit >=0 timestamp, lo extra sec bit on ok 8 - 2174-02-25 Upper bound of 32bit >=0 timestamp, lo extra sec bit on ok 9 - 2174-02-25 Lower bound of 32bit <0 timestamp, hi extra sec bit on ok 10 - 2242-03-16 Upper bound of 32bit <0 timestamp, hi extra sec bit on ok 11 - 2242-03-16 Lower bound of 32bit >=0 timestamp, hi extra sec bit on ok 12 - 2310-04-04 Upper bound of 32bit >=0 timestamp, hi extra sec bit on ok 13 - 2310-04-04 Upper bound of 32bit>=0 timestamp, hi extra sec bit 1. 1 ns ok 14 - 2378-04-22 Lower bound of 32bit>= timestamp. Extra sec bits 1. Max ns ok 15 - 2378-04-22 Lower bound of 32bit >=0 timestamp. All extra sec bits on ok 16 - 2446-05-10 Upper bound of 32bit >=0 timestamp. All extra sec bits on # inode_test_xtimestamp_decoding: pass:16 fail:0 skip:0 total:16 ok 1 - inode_test_xtimestamp_decoding

Signed-off-by: David Gow davidgow@google.com Reviewed-by: Daniel Latypov dlatypov@google.com --- lib/kunit/test.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/kunit/test.c b/lib/kunit/test.c index e028d98e4f5b..fe2ab31b5949 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -511,6 +511,8 @@ int kunit_run_tests(struct kunit_suite *suite) /* Get initial param. */ param_desc[0] = '\0'; test.param_value = test_case->generate_params(NULL, param_desc); + kunit_log(KERN_INFO, &test, KUNIT_SUBTEST_INDENT KUNIT_SUBTEST_INDENT + "# Subtest: %s", test_case->name); }

while (test.param_value) { @@ -523,9 +525,8 @@ int kunit_run_tests(struct kunit_suite *suite) }

kunit_log(KERN_INFO, &test, - KUNIT_SUBTEST_INDENT - "# %s: %s %d - %s", - test_case->name, + KUNIT_SUBTEST_INDENT KUNIT_SUBTEST_INDENT + "%s %d - %s", kunit_status_to_ok_not_ok(test.status), test.param_index + 1, param_desc);

-- 2.33.0.1079.g6e70778dc9-goog

Daniel Latypov

10:12 p.m.

On Tue, Oct 26, 2021 at 6:37 PM David Gow davidgow@google.com wrote:

...

The (K)TAP spec encourages test output to begin with a 'test plan': a count of the number of tests being run of the form: 1..n

However, some test suites might not know the number of subtests in advance (for example, KUnit's parameterised tests use a generator function). In this case, it's not possible to print the test plan in advance.

kunit_tool already parses test output which doesn't contain a plan, but reports an error. Since we want to use nested subtests with KUnit paramterised tests, remove this error.

Signed-off-by: David Gow davidgow@google.com

Reviewed-by: Daniel Latypov dlatypov@google.com

This looks to be unchanged from v1. Looks good to me given kunit itself will report SKIPPED for parameterised tests.

...

tools/testing/kunit/kunit_parser.py | 5 ++--- tools/testing/kunit/kunit_tool_test.py | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 3355196d0515..50ded55c168c 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -340,8 +340,8 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: """ Parses test plan line and stores the expected number of subtests in test object. Reports an error if expected count is 0.
  Returns False and reports missing test plan error if fails to parse
  test plan.
  Returns False and sets expected_count to None if there is no valid test
  plan.

  Accepted format:
  - '1..[number of subtests]'
@@ -356,7 +356,6 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: match = TEST_PLAN.match(lines.peek()) if not match: test.expected_count = None
          test.add_error('missing plan line!')
          return False
  test.log.append(lines.pop())
  expected_count = int(match.group(1))
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 9c4126731457..bc8793145713 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -191,7 +191,10 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines( file.readlines()))
          self.assertEqual(2, result.test.counts.errors)
          # A missing test plan is not an error.
          self.assertEqual(0, result.test.counts.errors)
          # All tests should be accounted for.
          self.assertEqual(10, result.test.counts.total())
          self.assertEqual(
                  kunit_parser.TestStatus.SUCCESS,
                  result.status)
-- 2.33.0.1079.g6e70778dc9-goog

David Gow

11:02 p.m.

On Thu, Oct 28, 2021 at 6:13 AM Daniel Latypov dlatypov@google.com wrote:

...

On Tue, Oct 26, 2021 at 6:37 PM David Gow davidgow@google.com wrote:

...
The (K)TAP spec encourages test output to begin with a 'test plan': a count of the number of tests being run of the form: 1..n

However, some test suites might not know the number of subtests in advance (for example, KUnit's parameterised tests use a generator function). In this case, it's not possible to print the test plan in advance.

kunit_tool already parses test output which doesn't contain a plan, but reports an error. Since we want to use nested subtests with KUnit paramterised tests, remove this error.

Signed-off-by: David Gow davidgow@google.com

Reviewed-by: Daniel Latypov dlatypov@google.com

This looks to be unchanged from v1.

Yeah, the changes in v2 were all in the new patches.

-- David

922

days inactive

922

days old

linux-kselftest-mirror@lists.linaro.org

6 comments

participants

tags (0)

participants (2)

Daniel Latypov
David Gow