Plumb XFAIL conditions up into TAP reporting and test counts.
Cc: Shuah Khan shuah@kernel.org Cc: Andy Lutomirski luto@amacapital.net Cc: Will Drewry wad@chromium.org Cc: linux-kselftest@vger.kernel.org Signed-off-by: Kees Cook keescook@chromium.org --- tools/testing/selftests/kselftest.h | 17 ++++++- tools/testing/selftests/kselftest_harness.h | 54 ++++++++++++++++----- 2 files changed, 58 insertions(+), 13 deletions(-)
diff --git a/tools/testing/selftests/kselftest.h b/tools/testing/selftests/kselftest.h index 3f0d236ca2e4..9b4efdbb07f6 100644 --- a/tools/testing/selftests/kselftest.h +++ b/tools/testing/selftests/kselftest.h @@ -17,12 +17,13 @@ * * ksft_print_msg(fmt, ...); * - * and finally report the pass/fail/skip state of the test with one of: + * and finally report the pass/fail/skip/xfail state of the test with one of: * * ksft_test_result(condition, fmt, ...); * ksft_test_result_pass(fmt, ...); * ksft_test_result_fail(fmt, ...); * ksft_test_result_skip(fmt, ...); + * ksft_test_result_xfail(fmt, ...); * ksft_test_result_error(fmt, ...); * * When all tests are finished, clean up and exit the program with one of: @@ -163,7 +164,19 @@ static inline void ksft_test_result_fail(const char *msg, ...) ksft_test_result_fail(fmt, ##__VA_ARGS__);\ } while (0)
-/* TODO: add ksft_test_result_xfail() */ +static inline void ksft_test_result_xfail(const char *msg, ...) +{ + int saved_errno = errno; + va_list args; + + ksft_cnt.ksft_xfail++; + + va_start(args, msg); + printf("ok %d # XFAIL ", ksft_test_num()); + errno = saved_errno; + vprintf(msg, args); + va_end(args); +}
static inline void ksft_test_result_skip(const char *msg, ...) { diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index f8f7e47c739a..6b06930468e5 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -126,8 +126,8 @@ fprintf(TH_LOG_STREAM, "# XFAIL " fmt "\n", \ ##__VA_ARGS__); \ } \ - /* TODO: find a way to pass xfail to test runner process. */ \ _metadata->passed = 1; \ + _metadata->xfail = 1; \ _metadata->trigger = 0; \ statement; \ } while (0) @@ -777,6 +777,7 @@ struct __test_metadata { struct __fixture_metadata *fixture; int termsig; int passed; + int xfail; /* did XFAIL get used? */ int trigger; /* extra handler after the evaluation */ int timeout; /* seconds to wait for test timeout */ bool timed_out; /* did this test timeout instead of exiting? */ @@ -866,17 +867,31 @@ void __wait_for_test(struct __test_metadata *t) fprintf(TH_LOG_STREAM, "# %s: Test terminated by timeout\n", t->name); } else if (WIFEXITED(status)) { - t->passed = t->termsig == -1 ? !WEXITSTATUS(status) : 0; if (t->termsig != -1) { + t->passed = 0; fprintf(TH_LOG_STREAM, "# %s: Test exited normally instead of by signal (code: %d)\n", t->name, WEXITSTATUS(status)); - } else if (!t->passed) { - fprintf(TH_LOG_STREAM, - "# %s: Test failed at step #%d\n", - t->name, - WEXITSTATUS(status)); + } else { + switch (WEXITSTATUS(status)) { + /* Success */ + case 0: + t->passed = 1; + break; + /* XFAIL */ + case 255: + t->passed = 1; + t->xfail = 1; + break; + /* Other failure, assume step report. */ + default: + t->passed = 0; + fprintf(TH_LOG_STREAM, + "# %s: Test failed at step #%d\n", + t->name, + WEXITSTATUS(status)); + } } } else if (WIFSIGNALED(status)) { t->passed = 0; @@ -906,6 +921,7 @@ void __run_test(struct __fixture_metadata *f, { /* reset test struct */ t->passed = 1; + t->xfail = 0; t->trigger = 0; t->step = 0; t->no_print = 0; @@ -918,15 +934,31 @@ void __run_test(struct __fixture_metadata *f, t->passed = 0; } else if (t->pid == 0) { t->fn(t, variant); - /* return the step that failed or 0 */ - _exit(t->passed ? 0 : t->step); + /* Make sure step doesn't get lost in reporting */ + if (t->step >= 255) { + ksft_print_msg("Too many test steps (%u)!?\n", t->step); + t->step = 254; + } + /* Use 255 for XFAIL */ + if (t->xfail) + _exit(255); + /* Pass is exit 0 */ + if (t->passed) + _exit(0); + /* Something else happened, report the step. */ + _exit(t->step); } else { __wait_for_test(t); } ksft_print_msg(" %4s %s%s%s.%s\n", t->passed ? "OK" : "FAIL", f->name, variant->name[0] ? "." : "", variant->name, t->name); - ksft_test_result(t->passed, "%s%s%s.%s\n", - f->name, variant->name[0] ? "." : "", variant->name, t->name); + + if (t->xfail) + ksft_test_result_xfail("%s%s%s.%s\n", + f->name, variant->name[0] ? "." : "", variant->name, t->name); + else + ksft_test_result(t->passed, "%s%s%s.%s\n", + f->name, variant->name[0] ? "." : "", variant->name, t->name); }
static int test_harness_run(int __attribute__((unused)) argc,