Sorry for taking so long to get to this. It's been a busy couple of weeks.

On Thu, Jan 23, 2020 at 10:47 AM Alan Maguire alan.maguire@oracle.com wrote:

add debugfs support for displaying kunit test suite results; this is especially useful for module-loaded tests to allow disentangling of test result display from other dmesg events.

Signed-off-by: Alan Maguire alan.maguire@oracle.com

include/kunit/test.h | 21 ++++++++--- lib/kunit/Makefile | 3 +- lib/kunit/debugfs.c | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib/kunit/debugfs.h | 16 ++++++++ lib/kunit/test.c | 85 ++++++++++++++++++++++++++++++----------- 5 files changed, 202 insertions(+), 27 deletions(-) create mode 100644 lib/kunit/debugfs.c create mode 100644 lib/kunit/debugfs.h

diff --git a/include/kunit/test.h b/include/kunit/test.h index 2dfb550..37219b8a 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -125,6 +125,8 @@ struct kunit_case { bool success; };

+#define kunit_status2str(status) (status ? "ok" : "not ok")

nit: I would prefer kunit_status_to_string. Also, I would prefer this as a static inline function.

/**

• KUNIT_CASE - A helper for creating a &struct kunit_case

@@ -157,6 +159,9 @@ struct kunit_suite { int (*init)(struct kunit *test); void (*exit)(struct kunit *test); struct kunit_case *test_cases;

•   /* private - internal use only */
  

•   struct dentry *debugfs;
  

};

/** @@ -197,6 +202,15 @@ struct kunit {

int kunit_run_tests(struct kunit_suite *suite);

+size_t kunit_suite_num_test_cases(struct kunit_suite *suite);

+unsigned int kunit_test_case_num(struct kunit_suite *suite,

•                            struct kunit_case *test_case);
  

+int __kunit_test_suites_init(struct kunit_suite **suites);

+void __kunit_test_suites_exit(struct kunit_suite **suites);

/**

• kunit_test_suites() - used to register one or more &struct kunit_suite

•                  with KUnit.
  

@@ -226,15 +240,12 @@ struct kunit { static struct kunit_suite *suites[] = { __VA_ARGS__, NULL}; \ static int kunit_test_suites_init(void) \ { \

•           unsigned int i;                                         \
  

•           for (i = 0; suites[i] != NULL; i++)                     \
  

•                   kunit_run_tests(suites[i]);                     \
  

•           return 0;                                               \
  

•           return __kunit_test_suites_init(suites);                \
    }                                                               \
    late_initcall(kunit_test_suites_init);                          \
    static void __exit kunit_test_suites_exit(void)                 \
    {                                                               \
  

•           return;                                                 \
  

•           return __kunit_test_suites_exit(suites);                \
    }                                                               \
    module_exit(kunit_test_suites_exit)
  
  

diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile index fab5564..869aab0 100644 --- a/lib/kunit/Makefile +++ b/lib/kunit/Makefile @@ -3,7 +3,8 @@ obj-$(CONFIG_KUNIT) += kunit.o kunit-objs += test.o \ string-stream.o \ assert.o \

•                                   try-catch.o
  

•                                   try-catch.o \
  

•                                   debugfs.o
  
  

obj-$(CONFIG_KUNIT_TEST) += kunit-test.o

diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c new file mode 100644 index 0000000..578843c --- /dev/null +++ b/lib/kunit/debugfs.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/*

• • Copyright (c) 2020, Oracle and/or its affiliates.
• • Author: Alan Maguire alan.maguire@oracle.com
• */

+#include <linux/debugfs.h> +#include <linux/module.h>

+#include <kunit/test.h>

+#include "string-stream.h"

+#define KUNIT_DEBUGFS_ROOT "kunit" +#define KUNIT_DEBUGFS_RESULTS "results"

+/*

• • Create a debugfs representation of test suites:
• • Path Semantics
• • /sys/kernel/debug/kunit/<testsuite>/results Show results of last run for

• •                                         testsuite
    

• */

+static struct dentry *debugfs_rootdir;

+void kunit_debugfs_cleanup(void) +{

•   debugfs_remove_recursive(debugfs_rootdir);
  

+}

+void kunit_debugfs_init(void) +{

•   if (!debugfs_rootdir)
  

•           debugfs_rootdir = debugfs_create_dir(KUNIT_DEBUGFS_ROOT, NULL);
  

+}

+static void debugfs_print_result(struct seq_file *seq,

•                            struct kunit_suite *suite,
  

•                            struct kunit_case *test_case)
  

+{

•   if (!test_case)
  

•           return;
  

•   seq_printf(seq, "\t%s %d - %s\n", kunit_status2str(test_case->success),
  

•              kunit_test_case_num(suite, test_case), test_case->name);
  

Do you think we might want to collect info printed to kunit_{info|warn|err}? I would think the expectation failure line at least would be desirable, no?

+}

+/*

• • /sys/kernel/debug/kunit/<testsuite>/results shows all results for testsuite.
• */

+static int debugfs_print_results(struct seq_file *seq, void *v) +{

•   struct kunit_suite *suite = (struct kunit_suite *)seq->private;
  

•   struct kunit_case *test_case;
  

•   if (!suite)
  

•           return 0;
  

•   seq_printf(seq, "\t# Subtest: %s\n", suite->name);
  

•   seq_printf(seq, "\t1..%zd\n", kunit_suite_num_test_cases(suite));
  

Is there any way to reuse the format strings/printing from test.c? It would be nice if we didn't have to remember to update two printers.

•   for (test_case = suite->test_cases; test_case->run_case; test_case++)
  

•           debugfs_print_result(seq, suite, test_case);
  

•   return 0;
  

+}

+static int debugfs_release(struct inode *inode, struct file *file) +{

•   return single_release(inode, file);
  

+}

+static int debugfs_results_open(struct inode *inode, struct file *file) +{

•   struct kunit_suite *suite;
  

•   suite = (struct kunit_suite *)inode->i_private;
  

•   return single_open(file, debugfs_print_results, suite);
  

+}

+static const struct file_operations debugfs_results_fops = {

•   .open = debugfs_results_open,
  

•   .read = seq_read,
  

•   .llseek = seq_lseek,
  

•   .release = debugfs_release,
  

+};

+void kunit_debugfs_create_suite(struct kunit_suite *suite) +{

•   /* First add /sys/kernel/debug/kunit/<testsuite> */
  

•   suite->debugfs = debugfs_create_dir(suite->name, debugfs_rootdir);
  

•   debugfs_create_file(KUNIT_DEBUGFS_RESULTS, S_IFREG | 0444,
  

•                       suite->debugfs,
  

•                       suite, &debugfs_results_fops);
  

+}

+void kunit_debugfs_destroy_suite(struct kunit_suite *suite) +{

•   debugfs_remove_recursive(suite->debugfs);
  

+} diff --git a/lib/kunit/debugfs.h b/lib/kunit/debugfs.h new file mode 100644 index 0000000..9a92d5e --- /dev/null +++ b/lib/kunit/debugfs.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/*

• • Copyright (C) 2020, Oracle and/or its affiliates.
• */

+#ifndef _KUNIT_DEBUGFS_H +#define _KUNIT_DEBUGFS_H

+#include <kunit/test.h>

+void kunit_debugfs_create_suite(struct kunit_suite *suite); +void kunit_debugfs_destroy_suite(struct kunit_suite *suite); +void kunit_debugfs_init(void); +void kunit_debugfs_cleanup(void);

+#endif /* _KUNIT_DEBUGFS_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 9242f93..07089ca 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -10,6 +10,7 @@ #include <linux/kernel.h> #include <linux/sched/debug.h>

+#include "debugfs.h" #include "string-stream.h" #include "try-catch-impl.h"

@@ -28,22 +29,24 @@ static void kunit_print_tap_version(void) } }

-static size_t kunit_test_cases_len(struct kunit_case *test_cases) +size_t kunit_suite_num_test_cases(struct kunit_suite *suite) {

•   struct kunit_case *test_case;
  

•   struct kunit_case *test_case, *test_cases = suite->test_cases;
    size_t len = 0;
  
  

•   for (test_case = test_cases; test_case->run_case; test_case++)
  

•   for (test_case = test_cases; test_case && test_case->run_case;
  

•        test_case++)
            len++;
  
    return len;
  

} +EXPORT_SYMBOL_GPL(kunit_suite_num_test_cases);

static void kunit_print_subtest_start(struct kunit_suite *suite) { kunit_print_tap_version(); pr_info("\t# Subtest: %s\n", suite->name);

•   pr_info("\t1..%zd\n", kunit_test_cases_len(suite->test_cases));
  

•   pr_info("\t1..%zd\n", kunit_suite_num_test_cases(suite));
  

}

static void kunit_print_ok_not_ok(bool should_indent, @@ -51,19 +54,15 @@ static void kunit_print_ok_not_ok(bool should_indent, size_t test_number, const char *description) {

•   const char *indent, *ok_not_ok;
  

•   const char *indent;
  
    if (should_indent)
            indent = "\t";
    else
            indent = "";
  
  

•   if (is_ok)
  

•           ok_not_ok = "ok";
  

•   else
  

•           ok_not_ok = "not ok";
  

•   pr_info("%s%s %zd - %s\n", indent, ok_not_ok, test_number, description);
  

•   pr_info("%s%s %zd - %s\n", indent, kunit_status2str(is_ok),
  

•           test_number, description);
  

}

static bool kunit_suite_has_succeeded(struct kunit_suite *suite) @@ -87,14 +86,20 @@ static void kunit_print_subtest_end(struct kunit_suite *suite) suite->name); }

-static void kunit_print_test_case_ok_not_ok(struct kunit_case *test_case,

•                                       size_t test_number)
  

+unsigned int kunit_test_case_num(struct kunit_suite *suite,

•                            struct kunit_case *test_case)
  

{

•   kunit_print_ok_not_ok(true,
  

•                         test_case->success,
  

•                         test_number,
  

•                         test_case->name);
  

•   struct kunit_case *tc;
  

•   unsigned int i;
  

•   for (i = 1, tc = suite->test_cases; tc->run_case; tc++, i++) {
  

•           if (tc == test_case)
  

•                   return i;
  

•   }
  

•   return 0;
  

} +EXPORT_SYMBOL_GPL(kunit_test_case_num);

static void kunit_print_string_stream(struct kunit *test, struct string_stream *stream) @@ -102,6 +107,9 @@ static void kunit_print_string_stream(struct kunit *test, struct string_stream_fragment *fragment; char *buf;

•   if (string_stream_is_empty(stream))
  

•           return;
  

•   buf = string_stream_get_string(stream);
    if (!buf) {
            kunit_err(test,
  

@@ -303,19 +311,20 @@ static void kunit_run_case_catch_errors(struct kunit_suite *suite, kunit_try_catch_run(try_catch, &context);

    test_case->success = test.success;

•   kunit_print_ok_not_ok(true, test_case->success,
  

•                         kunit_test_case_num(suite, test_case),
  

•                         test_case->name);
  

}

int kunit_run_tests(struct kunit_suite *suite) { struct kunit_case *test_case;

•   size_t test_case_count = 1;
  
    kunit_print_subtest_start(suite);
  
  

•   for (test_case = suite->test_cases; test_case->run_case; test_case++) {
  

•   for (test_case = suite->test_cases; test_case->run_case; test_case++)
            kunit_run_case_catch_errors(suite, test_case);
  

•           kunit_print_test_case_ok_not_ok(test_case, test_case_count++);
  

•   }
  
    kunit_print_subtest_end(suite);
  
  

@@ -323,6 +332,37 @@ int kunit_run_tests(struct kunit_suite *suite) } EXPORT_SYMBOL_GPL(kunit_run_tests);

+static void kunit_init_suite(struct kunit_suite *suite) +{

•   kunit_debugfs_create_suite(suite);
  

+}

It looks like this is only called in one place, can we just call kunit_debugfs_create_suite directly?

+int __kunit_test_suites_init(struct kunit_suite **suites) +{

•   unsigned int i;
  

•   for (i = 0; suites[i] != NULL; i++) {
  

•           kunit_init_suite(suites[i]);
  

•           kunit_run_tests(suites[i]);
  

•   }
  

•   return 0;
  

+} +EXPORT_SYMBOL_GPL(__kunit_test_suites_init);

+static void kunit_exit_suite(struct kunit_suite *suite) +{

•   kunit_debugfs_destroy_suite(suite);
  

+}

+void __kunit_test_suites_exit(struct kunit_suite **suites) +{

•   unsigned int i;
  

•   for (i = 0; suites[i] != NULL; i++)
  

•           kunit_exit_suite(suites[i]);
  

+} +EXPORT_SYMBOL_GPL(__kunit_test_suites_exit);

struct kunit_resource *kunit_alloc_and_get_resource(struct kunit *test, kunit_resource_init_t init, kunit_resource_free_t free, @@ -489,12 +529,15 @@ void kunit_cleanup(struct kunit *test)

static int __init kunit_init(void) {

•   kunit_debugfs_init();
  

•   return 0;
  

} late_initcall(kunit_init);

static void __exit kunit_exit(void) {

•   kunit_debugfs_cleanup();
  

} module_exit(kunit_exit);

-- 1.8.3.1

On Thu, Jan 23, 2020 at 10:47 AM Alan Maguire alan.maguire@oracle.com wrote:

Add /sys/kernel/debug/kunit/<suite>/run file which will run the specified suite and show results.

Signed-off-by: Alan Maguire alan.maguire@oracle.com

If you don't mind, I would like to see the device tree unit test from Frank before we accept this patch. I definitely like your approach here, but this would break with KUnit test cases which depend on __init code and data. I just figure that it would be easier for us to solve the __init problem now if we have a working example that uses it rather than having someone who wants to write a test which depends on __init having to fix this after the fact. Let me know if this is a problem for you.

---

lib/kunit/debugfs.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+)

diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c index 578843c..1ea3fbc 100644 --- a/lib/kunit/debugfs.c +++ b/lib/kunit/debugfs.c @@ -13,6 +13,7 @@

#define KUNIT_DEBUGFS_ROOT "kunit" #define KUNIT_DEBUGFS_RESULTS "results" +#define KUNIT_DEBUGFS_RUN "run"

/*

• Create a debugfs representation of test suites:

@@ -20,6 +21,7 @@

• Path Semantics
• /sys/kernel/debug/kunit/<testsuite>/results Show results of last run for

•                                         testsuite
  

• • /sys/kernel/debug/kunit/<testsuite>/run Run testsuite and show results
  */

@@ -67,6 +69,18 @@ static int debugfs_print_results(struct seq_file *seq, void *v) return 0; }

+/*

• • /sys/kernel/debug/kunit/<testsuite>/run (re)runs suite and shows all results.
• */

+static int debugfs_run_print_results(struct seq_file *seq, void *v) +{

•   struct kunit_suite *suite = (struct kunit_suite *)seq->private;
  

•   kunit_run_tests(suite);
  

•   return debugfs_print_results(seq, v);
  

+}

static int debugfs_release(struct inode *inode, struct file *file) { return single_release(inode, file); @@ -88,6 +102,22 @@ static int debugfs_results_open(struct inode *inode, struct file *file) .release = debugfs_release, };

+static int debugfs_run_open(struct inode *inode, struct file *file) +{

•   struct kunit_suite *suite;
  

•   suite = (struct kunit_suite *)inode->i_private;
  

•   return single_open(file, debugfs_run_print_results, suite);
  

+}

+static const struct file_operations debugfs_run_fops = {

•   .open = debugfs_run_open,
  

•   .read = seq_read,
  

•   .llseek = seq_lseek,
  

•   .release = debugfs_release,
  

+};

void kunit_debugfs_create_suite(struct kunit_suite *suite) { /* First add /sys/kernel/debug/kunit/<testsuite> */ @@ -96,6 +126,9 @@ void kunit_debugfs_create_suite(struct kunit_suite *suite) debugfs_create_file(KUNIT_DEBUGFS_RESULTS, S_IFREG | 0444, suite->debugfs, suite, &debugfs_results_fops);

•   debugfs_create_file(KUNIT_DEBUGFS_RUN, S_IFREG | 0444,
  

•                       suite->debugfs,
  

•                       suite, &debugfs_run_fops);
  

Should anyone be able to read this? I think I agree since I am of the opinion that people shouldn't build or load tests into a production environment, but still I think it should be brought up.

I was actually talking to David the other day and we had the idea that maybe KUnit should taint the kernel after tests run or after a failure. Maybe that might communicate to a user that after running tests the kernel shouldn't be used for production purposes. (Obviously, I don't expect you to make that change here, the point of anyone being able to cause tests to run just made me think of it.) What do you think?

}

void kunit_debugfs_destroy_suite(struct kunit_suite *suite)

1.8.3.1

On Thu, Jan 23, 2020 at 10:47 AM Alan Maguire alan.maguire@oracle.com wrote:

Documentation should describe debugfs layout and semantics.

Signed-off-by: Alan Maguire alan.maguire@oracle.com

Documentation/dev-tools/kunit/usage.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+)

diff --git a/Documentation/dev-tools/kunit/usage.rst b/Documentation/dev-tools/kunit/usage.rst index 7cd56a1..b05c843 100644 --- a/Documentation/dev-tools/kunit/usage.rst +++ b/Documentation/dev-tools/kunit/usage.rst @@ -590,3 +590,22 @@ able to run one test case per invocation.

.. TODO(brendanhiggins@google.com): Add an actual example of an architecture dependent KUnit test.

+KUnit debugfs representation +============================ +When kunit test suites are initialized, they create an associated directory +in /sys/kernel/debug/kunit/<test-suite>. The directory contains two files

+- results: "cat results" displays results of last test run +- run: "cat run" runs the test suite and displays the results

+Thus to re-run all currently loaded suites and display results, we can do this:

+``` +$ cat /sys/kernel/debug/kunit/*/run +```

This should be in a ".. code-block:: bash", see above in this file for an example.

+The debugfs representation is primarily of use when kunit test suites are +run in a native environment, either as modules or builtin. Having a way +to display results like this is valuable as otherwise results can be

+intermixed with other events in dmesg output.

1.8.3.1