This series demonstrates how KTAP output can be used by nolibc-test to make the test results better to read for people and machines. Especially when running multiple invocations for different architectors or build configurations we can make use of the kernels TAP parser to automatically provide aggregated test reports.
The code is very hacky and incomplete and mostly meant to validate if the output format is useful.
Start with the last patch of the series to actually see the generated format, or run it for yourself.
Signed-off-by: Thomas Weißschuh linux@weissschuh.net --- Thomas Weißschuh (7): selftests/nolibc: statically calculate number of testsuites selftests/nolibc: use unsigned indices for testcases selftests/nolibc: replace repetitive test structure with macro selftests/nolibc: count subtests kselftest: support KTAP format kselftest: support skipping tests with testname selftests/nolibc: proof of concept for TAP output
tools/testing/selftests/kselftest.h | 20 +++ tools/testing/selftests/nolibc/nolibc-test.c | 197 ++++++++++-------------- tools/testing/selftests/nolibc/run-all-tests.sh | 22 +++ 3 files changed, 127 insertions(+), 112 deletions(-) --- base-commit: dfef4fc45d5713eb23d87f0863aff9c33bd4bfaf change-id: 20230718-nolibc-ktap-tmp-4408f505408d
Best regards,
This makes it slightly nicer to report the number of suites.
Signed-off-by: Thomas Weißschuh linux@weissschuh.net --- tools/testing/selftests/nolibc/nolibc-test.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c index 03b1d30f5507..698af242d3e4 100644 --- a/tools/testing/selftests/nolibc/nolibc-test.c +++ b/tools/testing/selftests/nolibc/nolibc-test.c @@ -45,6 +45,8 @@ #define SINT_MAX_OF_TYPE(type) (((type)1 << (sizeof(type) * 8 - 2)) - (type)1 + ((type)1 << (sizeof(type) * 8 - 2))) #define SINT_MIN_OF_TYPE(type) (-SINT_MAX_OF_TYPE(type) - 1)
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) + /* will be used to test initialization of environ */ static char **test_envp;
@@ -1184,7 +1186,6 @@ static const struct test test_names[] = { { .name = "stdlib", .func = run_stdlib }, { .name = "vfprintf", .func = run_vfprintf }, { .name = "protection", .func = run_protection }, - { 0 } };
int is_setting_valid(char *test) @@ -1259,7 +1260,7 @@ int main(int argc, char **argv, char **envp) if (colon) *(colon++) = '\0';
- for (idx = 0; test_names[idx].name; idx++) { + for (idx = 0; idx < ARRAY_SIZE(test_names); idx++) { if (strcmp(test, test_names[idx].name) == 0) break; } @@ -1305,7 +1306,7 @@ int main(int argc, char **argv, char **envp) } while (test && *test); } else { /* no test mentioned, run everything */ - for (idx = 0; test_names[idx].name; idx++) { + for (idx = 0; idx < ARRAY_SIZE(test_names); idx++) { printf("Running test '%s'\n", test_names[idx].name); err = test_names[idx].func(min, max); ret += err;
They should never be negative.
Signed-off-by: Thomas Weißschuh linux@weissschuh.net --- tools/testing/selftests/nolibc/nolibc-test.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c index 698af242d3e4..1bd99e0fab4d 100644 --- a/tools/testing/selftests/nolibc/nolibc-test.c +++ b/tools/testing/selftests/nolibc/nolibc-test.c @@ -62,7 +62,7 @@ static const char *argv0; /* definition of a series of tests */ struct test { const char *name; /* test name */ - int (*func)(int min, int max); /* handler */ + int (*func)(unsigned int min, unsigned int max); /* handler */ };
#ifndef _NOLIBC_STDLIB_H @@ -570,7 +570,7 @@ static int expect_strne(const char *expr, int llen, const char *cmp) #define CASE_TEST(name) \ case __LINE__: llen += printf("%d %s", test, #name);
-int run_startup(int min, int max) +int run_startup(unsigned int min, unsigned int max) { int test; int ret = 0; @@ -773,7 +773,7 @@ int test_mmap_munmap(void) /* Run syscall tests between IDs <min> and <max>. * Return 0 on success, non-zero on failure. */ -int run_syscall(int min, int max) +int run_syscall(unsigned int min, unsigned int max) { struct timeval tv; struct timezone tz; @@ -884,7 +884,7 @@ int run_syscall(int min, int max) return ret; }
-int run_stdlib(int min, int max) +int run_stdlib(unsigned int min, unsigned int max) { int test; int tmp; @@ -1027,7 +1027,7 @@ static int expect_vfprintf(int llen, size_t c, const char *expected, const char return ret; }
-static int run_vfprintf(int min, int max) +static int run_vfprintf(unsigned int min, unsigned int max) { int test; int tmp; @@ -1070,7 +1070,7 @@ static int smash_stack(void) return 1; }
-static int run_protection(int min, int max) +static int run_protection(unsigned int min, unsigned int max) { pid_t pid; int llen = 0, status;
A future patch will change these, so prepare for that.
Signed-off-by: Thomas Weißschuh linux@weissschuh.net --- tools/testing/selftests/nolibc/nolibc-test.c | 35 ++++++++++++---------------- 1 file changed, 15 insertions(+), 20 deletions(-)
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c index 1bd99e0fab4d..76bd6a0a8132 100644 --- a/tools/testing/selftests/nolibc/nolibc-test.c +++ b/tools/testing/selftests/nolibc/nolibc-test.c @@ -570,6 +570,13 @@ static int expect_strne(const char *expr, int llen, const char *cmp) #define CASE_TEST(name) \ case __LINE__: llen += printf("%d %s", test, #name);
+#define SWITCH_TEST \ + switch (test + __LINE__ + 1) { + +#define SWITCH_TEST_END \ + case __LINE__: return ret; } + + int run_startup(unsigned int min, unsigned int max) { int test; @@ -591,7 +598,7 @@ int run_startup(unsigned int min, unsigned int max) /* avoid leaving empty lines below, this will insert holes into * test numbers. */ - switch (test + __LINE__ + 1) { + SWITCH_TEST CASE_TEST(argc); EXPECT_GE(1, test_argc, 1); break; CASE_TEST(argv_addr); EXPECT_PTRGT(1, test_argv, brk); break; CASE_TEST(argv_environ); EXPECT_PTRLT(1, test_argv, environ); break; @@ -607,10 +614,7 @@ int run_startup(unsigned int min, unsigned int max) CASE_TEST(auxv_addr); EXPECT_PTRGT(test_auxv != (void *)-1, test_auxv, brk); break; CASE_TEST(auxv_AT_UID); EXPECT_EQ(1, getauxval(AT_UID), getuid()); break; CASE_TEST(auxv_AT_PAGESZ); EXPECT_GE(1, getauxval(AT_PAGESZ), 4096); break; - case __LINE__: - return ret; /* must be last */ - /* note: do not set any defaults so as to permit holes above */ - } + SWITCH_TEST_END } return ret; } @@ -803,7 +807,7 @@ int run_syscall(unsigned int min, unsigned int max) /* avoid leaving empty lines below, this will insert holes into * test numbers. */ - switch (test + __LINE__ + 1) { + SWITCH_TEST CASE_TEST(getpid); EXPECT_SYSNE(1, getpid(), -1); break; CASE_TEST(getppid); EXPECT_SYSNE(1, getppid(), -1); break; CASE_TEST(gettid); EXPECT_SYSNE(has_gettid, gettid(), -1); break; @@ -876,10 +880,7 @@ int run_syscall(unsigned int min, unsigned int max) CASE_TEST(write_zero); EXPECT_SYSZR(1, write(1, &tmp, 0)); break; CASE_TEST(syscall_noargs); EXPECT_SYSEQ(1, syscall(__NR_getpid), getpid()); break; CASE_TEST(syscall_args); EXPECT_SYSER(1, syscall(__NR_statx, 0, NULL, 0, 0, NULL), -1, EFAULT); break; - case __LINE__: - return ret; /* must be last */ - /* note: do not set any defaults so as to permit holes above */ - } + SWITCH_TEST_END } return ret; } @@ -897,7 +898,7 @@ int run_stdlib(unsigned int min, unsigned int max) /* avoid leaving empty lines below, this will insert holes into * test numbers. */ - switch (test + __LINE__ + 1) { + SWITCH_TEST CASE_TEST(getenv_TERM); EXPECT_STRNZ(1, getenv("TERM")); break; CASE_TEST(getenv_blah); EXPECT_STRZR(1, getenv("blah")); break; CASE_TEST(setcmp_blah_blah); EXPECT_EQ(1, strcmp("blah", "blah"), 0); break; @@ -966,10 +967,7 @@ int run_stdlib(unsigned int min, unsigned int max) CASE_TEST(limit_ptrdiff_min_32); EXPECT_EQ(sizeof(long) == 4, PTRDIFF_MIN, (ptrdiff_t) 0x80000000); break; CASE_TEST(limit_ptrdiff_max_32); EXPECT_EQ(sizeof(long) == 4, PTRDIFF_MAX, (ptrdiff_t) 0x7fffffff); break; CASE_TEST(limit_size_max_32); EXPECT_EQ(sizeof(long) == 4, SIZE_MAX, (size_t) 0xffffffffU); break; - case __LINE__: - return ret; /* must be last */ - /* note: do not set any defaults so as to permit holes above */ - } + SWITCH_TEST_END } return ret; } @@ -1040,7 +1038,7 @@ static int run_vfprintf(unsigned int min, unsigned int max) /* avoid leaving empty lines below, this will insert holes into * test numbers. */ - switch (test + __LINE__ + 1) { + SWITCH_TEST CASE_TEST(empty); EXPECT_VFPRINTF(0, "", ""); break; CASE_TEST(simple); EXPECT_VFPRINTF(3, "foo", "foo"); break; CASE_TEST(string); EXPECT_VFPRINTF(3, "foo", "%s", "foo"); break; @@ -1050,10 +1048,7 @@ static int run_vfprintf(unsigned int min, unsigned int max) CASE_TEST(char); EXPECT_VFPRINTF(1, "c", "%c", 'c'); break; CASE_TEST(hex); EXPECT_VFPRINTF(1, "f", "%x", 0xf); break; CASE_TEST(pointer); EXPECT_VFPRINTF(3, "0x1", "%p", (void *) 0x1); break; - case __LINE__: - return ret; /* must be last */ - /* note: do not set any defaults so as to permit holes above */ - } + SWITCH_TEST_END } return ret; }
This will be needed for KTAP output.
Signed-off-by: Thomas Weißschuh linux@weissschuh.net --- tools/testing/selftests/nolibc/nolibc-test.c | 30 +++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-)
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c index 76bd6a0a8132..03f64ce1dda6 100644 --- a/tools/testing/selftests/nolibc/nolibc-test.c +++ b/tools/testing/selftests/nolibc/nolibc-test.c @@ -65,6 +65,13 @@ struct test { int (*func)(unsigned int min, unsigned int max); /* handler */ };
+#define COUNT_SUBTESTS (-1) + +static unsigned int count_subtests(const struct test *test) +{ + return test->func(COUNT_SUBTESTS, COUNT_SUBTESTS); +} + #ifndef _NOLIBC_STDLIB_H char *itoa(int i) { @@ -571,10 +578,10 @@ static int expect_strne(const char *expr, int llen, const char *cmp) case __LINE__: llen += printf("%d %s", test, #name);
#define SWITCH_TEST \ - switch (test + __LINE__ + 1) { + int _tests_start = __LINE__; switch (test + __LINE__ + 1) {
#define SWITCH_TEST_END \ - case __LINE__: return ret; } + case __LINE__: return ret; default: return __LINE__ - _tests_start - 1; }
int run_startup(unsigned int min, unsigned int max) @@ -592,7 +599,7 @@ int run_startup(unsigned int min, unsigned int max) test_auxv = _auxv; #endif
- for (test = min; test >= 0 && test <= max; test++) { + for (test = min; test == COUNT_SUBTESTS || (test >= 0 && test <= max); test++) { int llen = 0; /* line length */
/* avoid leaving empty lines below, this will insert holes into @@ -801,7 +808,7 @@ int run_syscall(unsigned int min, unsigned int max) has_gettid = __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 30); #endif
- for (test = min; test >= 0 && test <= max; test++) { + for (test = min; test == COUNT_SUBTESTS || (test >= 0 && test <= max); test++) { int llen = 0; /* line length */
/* avoid leaving empty lines below, this will insert holes into @@ -892,7 +899,7 @@ int run_stdlib(unsigned int min, unsigned int max) int ret = 0; void *p1, *p2;
- for (test = min; test >= 0 && test <= max; test++) { + for (test = min; test == COUNT_SUBTESTS || (test >= 0 && test <= max); test++) { int llen = 0; /* line length */
/* avoid leaving empty lines below, this will insert holes into @@ -1032,7 +1039,7 @@ static int run_vfprintf(unsigned int min, unsigned int max) int ret = 0; void *p1, *p2;
- for (test = min; test >= 0 && test <= max; test++) { + for (test = min; test == COUNT_SUBTESTS || (test >= 0 && test <= max); test++) { int llen = 0; /* line length */
/* avoid leaving empty lines below, this will insert holes into @@ -1070,6 +1077,9 @@ static int run_protection(unsigned int min, unsigned int max) pid_t pid; int llen = 0, status;
+ if (min == COUNT_SUBTESTS) + return 1; + llen += printf("0 -fstackprotector ");
#if !defined(_NOLIBC_STACKPROTECTOR) @@ -1219,6 +1229,7 @@ int main(int argc, char **argv, char **envp) int ret = 0; int err; int idx; + unsigned int subtests; char *test;
argv0 = argv[0]; @@ -1265,6 +1276,8 @@ int main(int argc, char **argv, char **envp) * once. We may have an optional range at <colon> * here, which defaults to the full range. */ + + subtests = count_subtests(&test_names[idx]); do { min = 0; max = INT_MAX; value = colon; @@ -1285,11 +1298,14 @@ int main(int argc, char **argv, char **envp) else if (*dash) max = atoi(dash);
+ if (max > subtests) + max = subtests; + value = colon; }
/* now's time to call the test */ - printf("Running test '%s'\n", test_names[idx].name); + printf("Running test '%s' (%d-%d of %d)\n", test_names[idx].name, min, max, subtests); err = test_names[idx].func(min, max); ret += err; printf("Errors during this test: %d\n\n", err);
KTAP supports nested suites which we want to use for nolibc tests. The TAP parser at tools/testing/kunit/kunit.py requires the header "KTAP version 1" to parse nested suites.
Signed-off-by: Thomas Weißschuh linux@weissschuh.net --- tools/testing/selftests/kselftest.h | 5 +++++ 1 file changed, 5 insertions(+)
diff --git a/tools/testing/selftests/kselftest.h b/tools/testing/selftests/kselftest.h index 829be379545a..811d720d50dd 100644 --- a/tools/testing/selftests/kselftest.h +++ b/tools/testing/selftests/kselftest.h @@ -117,6 +117,11 @@ static inline void ksft_print_header(void) printf("TAP version 13\n"); }
+static inline void ksft_print_header_ktap(void) +{ + printf("KTAP version 1\n"); +} + static inline void ksft_set_plan(unsigned int plan) { ksft_plan = plan;
The TAP parser at tools/testing/kunit/kunit.py requires a testname to properly parse skipped tests. The current kselftest APIs do not provide this functionality so add it.
Signed-off-by: Thomas Weißschuh linux@weissschuh.net --- tools/testing/selftests/kselftest.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+)
diff --git a/tools/testing/selftests/kselftest.h b/tools/testing/selftests/kselftest.h index 811d720d50dd..0206287de5b8 100644 --- a/tools/testing/selftests/kselftest.h +++ b/tools/testing/selftests/kselftest.h @@ -219,6 +219,21 @@ static inline void ksft_test_result_skip(const char *msg, ...) va_end(args); }
+static inline void ksft_test_result_skip2(const char *name, const char *msg, ...) +{ + int saved_errno = errno; + va_list args; + + ksft_cnt.ksft_xskip++; + + va_start(args, msg); + printf("ok %d %s # SKIP ", ksft_test_num(), name); + errno = saved_errno; + vprintf(msg, args); + va_end(args); +} + + /* TODO: how does "error" differ from "fail" or "skip"? */ static inline void ksft_test_result_error(const char *msg, ...) {
Dirty proof of concept to show how (K)TAP output can look and how it can be used.
Currently test selection is not supported and for simplicity only the startup tests are enabled.
Example output:
$ ./nolibc-test KTAP version 1 1..15 ok 1 argc = 1 ok 2 argv_addr = <0x7ffdc66173a8> ok 3 argv_environ = <0x7ffdc66173a8> ok 4 argv_total = 1 ok 5 argv0_addr = <0x7ffdc6618bca> ok 6 argv0_str = <0x7ffdc6618bca> ok 7 argv0_len = 13 ok 8 environ_addr = <0x7ffdc66173b8> ok 9 environ_envp = <0x7ffdc66173b8> ok 10 environ_auxv = <0x7ffdc66173b8> ok 11 environ_total = 271 ok 12 environ_HOME = <0x7ffdc6618cc7> ok 13 auxv_addr = <0x7ffdc66174c8> ok 14 auxv_AT_UID = 1000 ok 15 auxv_AT_PAGESZ = 4096 # Exiting with status 0 # Totals: pass:15 fail:0 xfail:0 xpass:0 skip:0 error:0
$ ./libc-test KTAP version 1 1..15 ok 1 argc = 1 ok 2 argv_addr = <0x7ffd5f3d43e8> ok 3 argv_environ = <0x7ffd5f3d43e8> ok 4 argv_total = 1 ok 5 argv0_addr = <0x7ffd5f3d5bd0> ok 6 argv0_str = <0x7ffd5f3d5bd0> ok 7 argv0_len = 11 ok 8 environ_addr = <0x7ffd5f3d43f8> ok 9 environ_envp = <0x7ffd5f3d43f8> ok 10 environ_auxv # SKIP test_auxv != (void *)-1 ok 11 environ_total # SKIP test_auxv != (void *)-1 ok 12 environ_HOME = <0x7ffd5f3d5ccb> ok 13 auxv_addr # SKIP test_auxv != (void *)-1 ok 14 auxv_AT_UID = 1000 ok 15 auxv_AT_PAGESZ = 4096 # Exiting with status 0 # Totals: pass:12 fail:0 xfail:0 xpass:0 skip:3 error:0
./run-all-tests.sh | $SRC/tools/testing/kunit/kunit.py parse [23:47:26] ============================================================ [23:47:26] ====================== (15 subtests) ======================= [23:47:26] [PASSED] argc = 1 [23:47:26] [PASSED] argv_addr = <0x7ffcac1b8bc8> [23:47:26] [PASSED] argv_environ = <0x7ffcac1b8bc8> [23:47:26] [PASSED] argv_total = 1 [23:47:26] [PASSED] argv0_addr = <0x7ffcac1b9bd0> [23:47:26] [PASSED] argv0_str = <0x7ffcac1b9bd0> [23:47:26] [PASSED] argv0_len = 11 [23:47:26] [PASSED] environ_addr = <0x7ffcac1b8bd8> [23:47:26] [PASSED] environ_envp = <0x7ffcac1b8bd8> [23:47:26] [SKIPPED] environ_auxv [23:47:26] [SKIPPED] environ_total [23:47:26] [PASSED] environ_HOME = <0x7ffcac1b9ccb> [23:47:26] [SKIPPED] auxv_addr [23:47:26] [PASSED] auxv_AT_UID = 1000 [23:47:26] [PASSED] auxv_AT_PAGESZ = 4096 [23:47:26] ====================== [PASSED] arm64 ====================== [23:47:26] ====================== (15 subtests) ======================= [23:47:26] [PASSED] argc = 1 [23:47:26] [PASSED] argv_addr = <0x7ffdee178188> [23:47:26] [PASSED] argv_environ = <0x7ffdee178188> [23:47:26] [PASSED] argv_total = 1 [23:47:26] [PASSED] argv0_addr = <0x7ffdee178bd0> [23:47:26] [PASSED] argv0_str = <0x7ffdee178bd0> [23:47:26] [PASSED] argv0_len = 11 [23:47:26] [PASSED] environ_addr = <0x7ffdee178198> [23:47:26] [PASSED] environ_envp = <0x7ffdee178198> [23:47:26] [SKIPPED] environ_auxv [23:47:26] [SKIPPED] environ_total [23:47:26] [PASSED] environ_HOME = <0x7ffdee178ccb> [23:47:26] [SKIPPED] auxv_addr [23:47:26] [PASSED] auxv_AT_UID = 1000 [23:47:26] [PASSED] auxv_AT_PAGESZ = 4096 [23:47:26] ===================== [PASSED] x86_64 ====================== [23:47:26] ====================== (15 subtests) ======================= [23:47:26] [PASSED] argc = 1 [23:47:26] [PASSED] argv_addr = <0x7ffc16bf3628> [23:47:26] [PASSED] argv_environ = <0x7ffc16bf3628> [23:47:26] [PASSED] argv_total = 1 [23:47:26] [PASSED] argv0_addr = <0x7ffc16bf4bd0> [23:47:26] [PASSED] argv0_str = <0x7ffc16bf4bd0> [23:47:26] [PASSED] argv0_len = 11 [23:47:26] [PASSED] environ_addr = <0x7ffc16bf3638> [23:47:26] [PASSED] environ_envp = <0x7ffc16bf3638> [23:47:26] [SKIPPED] environ_auxv [23:47:26] [SKIPPED] environ_total [23:47:26] [PASSED] environ_HOME = <0x7ffc16bf4ccb> [23:47:26] [SKIPPED] auxv_addr [23:47:26] [PASSED] auxv_AT_UID = 1000 [23:47:26] [PASSED] auxv_AT_PAGESZ = 4096 [23:47:26] ===================== [PASSED] riscv64 ===================== [23:47:26] ============================================================ [23:47:26] Testing complete. Ran 45 tests: passed: 36, skipped: 9
The output of kunit.py is colored after the test results.
Not-signed-off --- tools/testing/selftests/nolibc/nolibc-test.c | 121 ++++++++---------------- tools/testing/selftests/nolibc/run-all-tests.sh | 22 +++++ 2 files changed, 63 insertions(+), 80 deletions(-)
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c index 03f64ce1dda6..f8064cd58783 100644 --- a/tools/testing/selftests/nolibc/nolibc-test.c +++ b/tools/testing/selftests/nolibc/nolibc-test.c @@ -41,12 +41,13 @@ #endif #endif
+#define inline __inline__ +#include "../kselftest.h" + /* for the type of int_fast16_t and int_fast32_t, musl differs from glibc and nolibc */ #define SINT_MAX_OF_TYPE(type) (((type)1 << (sizeof(type) * 8 - 2)) - (type)1 + ((type)1 << (sizeof(type) * 8 - 2))) #define SINT_MIN_OF_TYPE(type) (-SINT_MAX_OF_TYPE(type) - 1)
-#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) - /* will be used to test initialization of environ */ static char **test_envp;
@@ -59,6 +60,8 @@ static int test_argc; /* will be used by some test cases as readable file, please don't write it */ static const char *argv0;
+static const char *test_name; + /* definition of a series of tests */ struct test { const char *name; /* test name */ @@ -197,15 +200,11 @@ static int expect_nz(int expr, int llen)
#define EXPECT_EQ(cond, expr, val) \ - do { if (!(cond)) result(llen, SKIPPED); else ret += expect_eq(expr, llen, val); } while (0) + do { if (!(cond)) ksft_test_result_skip2(test_name, "%s\n", #cond); else expect_eq(expr, val); } while (0)
-static int expect_eq(uint64_t expr, int llen, uint64_t val) +static int expect_eq(uint64_t expr, uint64_t val) { - int ret = !(expr == val); - - llen += printf(" = %lld ", (long long)expr); - result(llen, ret ? FAIL : OK); - return ret; + ksft_test_result(expr = val, "%s = %lld\n", test_name, (long long) expr); }
@@ -223,15 +222,11 @@ static int expect_ne(int expr, int llen, int val)
#define EXPECT_GE(cond, expr, val) \ - do { if (!(cond)) result(llen, SKIPPED); else ret += expect_ge(expr, llen, val); } while (0) + do { if (!(cond)) ksft_test_result_skip2(test_name, "%s\n", #cond); else expect_ge(expr, val); } while (0)
-static int expect_ge(int expr, int llen, int val) +static int expect_ge(int expr, int val) { - int ret = !(expr >= val); - - llen += printf(" = %d ", expr); - result(llen, ret ? FAIL : OK); - return ret; + ksft_test_result(expr >= val, "%s = %d\n", test_name, expr); }
@@ -376,37 +371,19 @@ static int expect_ptrzr(const void *expr, int llen)
#define EXPECT_PTRNZ(cond, expr) \ - do { if (!(cond)) result(llen, SKIPPED); else ret += expect_ptrnz(expr, llen); } while (0) + do { if (!(cond)) ksft_test_result_skip("%s: %s\n", test_name, #cond); else expect_ptrnz(expr); } while (0)
-static int expect_ptrnz(const void *expr, int llen) +static int expect_ptrnz(const void *expr) { - int ret = 0; - - llen += printf(" = <%p> ", expr); - if (!expr) { - ret = 1; - result(llen, FAIL); - } else { - result(llen, OK); - } - return ret; + ksft_test_result(expr, "%s = <%p>\n", test_name, expr); }
#define EXPECT_PTREQ(cond, expr, cmp) \ - do { if (!(cond)) result(llen, SKIPPED); else ret += expect_ptreq(expr, llen, cmp); } while (0) + do { if (!(cond)) ksft_test_result_skip("%s: %s\n", test_name, #cond); else expect_ptreq(expr, cmp); } while (0)
-static int expect_ptreq(const void *expr, int llen, const void *cmp) +static void expect_ptreq(const void *expr, const void *cmp) { - int ret = 0; - - llen += printf(" = <%p> ", expr); - if (expr != cmp) { - ret = 1; - result(llen, FAIL); - } else { - result(llen, OK); - } - return ret; + ksft_test_result(expr == cmp, "%s = <%p>\n", test_name, expr); }
#define EXPECT_PTRNE(cond, expr, cmp) \ @@ -439,41 +416,28 @@ static int expect_ptrge(const void *expr, int llen, const void *cmp) }
#define EXPECT_PTRGT(cond, expr, cmp) \ - do { if (!(cond)) result(llen, SKIPPED); else ret += expect_ptrgt(expr, llen, cmp); } while (0) + do { if (!(cond)) ksft_test_result_skip2(test_name, "%s\n", #cond); else expect_ptrgt(expr, cmp); } while (0)
-static int expect_ptrgt(const void *expr, int llen, const void *cmp) +static void expect_ptrgt(const void *expr, const void *cmp) { - int ret = !(expr > cmp); - - llen += printf(" = <%p> ", expr); - result(llen, ret ? FAIL : OK); - return ret; + ksft_test_result(expr > cmp, "%s = <%p>\n", test_name, expr); }
- #define EXPECT_PTRLE(cond, expr, cmp) \ - do { if (!(cond)) result(llen, SKIPPED); else ret += expect_ptrle(expr, llen, cmp); } while (0) + do { if (!(cond)) ksft_test_result_skip("%s: %s\n", test_name, #cond); else expect_ptrle(expr, cmp); } while (0)
static int expect_ptrle(const void *expr, int llen, const void *cmp) { - int ret = !(expr <= cmp); - - llen += printf(" = <%p> ", expr); - result(llen, ret ? FAIL : OK); - return ret; + ksft_test_result(expr <= cmp, "%s = <%p>\n", test_name, expr); }
#define EXPECT_PTRLT(cond, expr, cmp) \ - do { if (!(cond)) result(llen, SKIPPED); else ret += expect_ptrlt(expr, llen, cmp); } while (0) + do { if (!(cond)) ksft_test_result_skip2(test_name, "%s\n", #cond); else expect_ptrlt(expr, cmp); } while (0)
-static int expect_ptrlt(const void *expr, int llen, const void *cmp) +static int expect_ptrlt(const void *expr, const void *cmp) { - int ret = !(expr < cmp); - - llen += printf(" = <%p> ", expr); - result(llen, ret ? FAIL : OK); - return ret; + ksft_test_result(expr < cmp, "%s = <%p>\n", test_name, expr); }
#define EXPECT_PTRER2(cond, expr, expret, experr1, experr2) \ @@ -520,20 +484,11 @@ static int expect_strzr(const char *expr, int llen)
#define EXPECT_STRNZ(cond, expr) \ - do { if (!(cond)) result(llen, SKIPPED); else ret += expect_strnz(expr, llen); } while (0) + do { if (!(cond)) ksft_test_result_skip("%s: %s\n", test_name, #cond); else expect_strnz(expr); } while (0)
-static int expect_strnz(const char *expr, int llen) +static int expect_strnz(const char *expr) { - int ret = 0; - - llen += printf(" = <%s> ", expr); - if (!expr) { - ret = 1; - result(llen, FAIL); - } else { - result(llen, OK); - } - return ret; + ksft_test_result(expr, "%s = <%p>\n", test_name, expr); }
@@ -575,7 +530,7 @@ static int expect_strne(const char *expr, int llen, const char *cmp)
/* declare tests based on line numbers. There must be exactly one test per line. */ #define CASE_TEST(name) \ - case __LINE__: llen += printf("%d %s", test, #name); + case __LINE__: test_name = #name;
#define SWITCH_TEST \ int _tests_start = __LINE__; switch (test + __LINE__ + 1) { @@ -1187,10 +1142,12 @@ int prepare(void) static const struct test test_names[] = { /* add new tests here */ { .name = "startup", .func = run_startup }, + /* { .name = "syscall", .func = run_syscall }, { .name = "stdlib", .func = run_stdlib }, { .name = "vfprintf", .func = run_vfprintf }, { .name = "protection", .func = run_protection }, + */ };
int is_setting_valid(char *test) @@ -1317,16 +1274,19 @@ int main(int argc, char **argv, char **envp) } while (test && *test); } else { /* no test mentioned, run everything */ + + ksft_print_header_ktap(); + int total = 0; + for (idx = 0; idx < ARRAY_SIZE(test_names); idx++) + total += count_subtests(&test_names[idx]); + ksft_set_plan(total); + for (idx = 0; idx < ARRAY_SIZE(test_names); idx++) { - printf("Running test '%s'\n", test_names[idx].name); err = test_names[idx].func(min, max); ret += err; - printf("Errors during this test: %d\n\n", err); } }
- printf("Total number of errors: %d\n", ret); - if (getpid() == 1) { /* we're running as init, there's no other process on the * system, thus likely started from a VM for a quick check. @@ -1335,7 +1295,7 @@ int main(int argc, char **argv, char **envp) * cleanly will often be reported as a success. This allows * to use the output of this program for bisecting kernels. */ - printf("Leaving init with final status: %d\n", !!ret); + ksft_print_msg("Leaving init with final status: %d\n", !!ret); if (ret == 0) reboot(RB_POWER_OFF); #if defined(__x86_64__) @@ -1349,6 +1309,7 @@ int main(int argc, char **argv, char **envp) #endif }
- printf("Exiting with status %d\n", !!ret); + ksft_print_msg("Exiting with status %d\n", !!ret); + ksft_finished(); return !!ret; } diff --git a/tools/testing/selftests/nolibc/run-all-tests.sh b/tools/testing/selftests/nolibc/run-all-tests.sh new file mode 100755 index 000000000000..8f37b8b36ef4 --- /dev/null +++ b/tools/testing/selftests/nolibc/run-all-tests.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -o pipefail + +archs=(arm64 x86_64 riscv64) + +echo "KTAP version 1" +echo "1..${#archs[@]}" + +for i in "${!archs[@]}"; do + arch="${archs[$i]}" + ./libc-test | sed -e 's/^/ /' + rc=$? + + if [ $rc -eq 0 ]; then + res=ok + else + res="not ok" + fi + + echo "$res $(( i + 1 )) $arch" +done
Hi, Thomas
The result looks very similar and the change seems not that big, thanks.
I have a generic question: do we need to compile nolibc-test.c independently or at least let users easily compile nolibc-test.c in the other places no just in kernel source code, for example, the other libcs may want to download and compile it directly.
The functions used in this change seems not many, is it able to provide our clones for them or only provide the clones when we compile them out-of-kernel.
for example:
#ifdef NOLIBC_TEST_IN_KERNEL /* -DNOLIBC_TEST_IN_KERNEL from Makefile, for future compatibility */ #include "../kselftest.h" #else // our clones of the used functions, for standalone usage #endif
Best regards, Zhangjin
Dirty proof of concept to show how (K)TAP output can look and how it can be used.
Currently test selection is not supported and for simplicity only the startup tests are enabled.
Example output:
$ ./nolibc-test KTAP version 1 1..15 ok 1 argc = 1 ok 2 argv_addr = <0x7ffdc66173a8> ok 3 argv_environ = <0x7ffdc66173a8> ok 4 argv_total = 1 ok 5 argv0_addr = <0x7ffdc6618bca> ok 6 argv0_str = <0x7ffdc6618bca> ok 7 argv0_len = 13 ok 8 environ_addr = <0x7ffdc66173b8> ok 9 environ_envp = <0x7ffdc66173b8> ok 10 environ_auxv = <0x7ffdc66173b8> ok 11 environ_total = 271 ok 12 environ_HOME = <0x7ffdc6618cc7> ok 13 auxv_addr = <0x7ffdc66174c8> ok 14 auxv_AT_UID = 1000 ok 15 auxv_AT_PAGESZ = 4096 # Exiting with status 0 # Totals: pass:15 fail:0 xfail:0 xpass:0 skip:0 error:0
$ ./libc-test KTAP version 1 1..15 ok 1 argc = 1 ok 2 argv_addr = <0x7ffd5f3d43e8> ok 3 argv_environ = <0x7ffd5f3d43e8> ok 4 argv_total = 1 ok 5 argv0_addr = <0x7ffd5f3d5bd0> ok 6 argv0_str = <0x7ffd5f3d5bd0> ok 7 argv0_len = 11 ok 8 environ_addr = <0x7ffd5f3d43f8> ok 9 environ_envp = <0x7ffd5f3d43f8> ok 10 environ_auxv # SKIP test_auxv != (void *)-1 ok 11 environ_total # SKIP test_auxv != (void *)-1 ok 12 environ_HOME = <0x7ffd5f3d5ccb> ok 13 auxv_addr # SKIP test_auxv != (void *)-1 ok 14 auxv_AT_UID = 1000 ok 15 auxv_AT_PAGESZ = 4096 # Exiting with status 0 # Totals: pass:12 fail:0 xfail:0 xpass:0 skip:3 error:0
./run-all-tests.sh | $SRC/tools/testing/kunit/kunit.py parse [23:47:26] ============================================================ [23:47:26] ====================== (15 subtests) ======================= [23:47:26] [PASSED] argc = 1 [23:47:26] [PASSED] argv_addr = <0x7ffcac1b8bc8> [23:47:26] [PASSED] argv_environ = <0x7ffcac1b8bc8> [23:47:26] [PASSED] argv_total = 1 [23:47:26] [PASSED] argv0_addr = <0x7ffcac1b9bd0> [23:47:26] [PASSED] argv0_str = <0x7ffcac1b9bd0> [23:47:26] [PASSED] argv0_len = 11 [23:47:26] [PASSED] environ_addr = <0x7ffcac1b8bd8> [23:47:26] [PASSED] environ_envp = <0x7ffcac1b8bd8> [23:47:26] [SKIPPED] environ_auxv [23:47:26] [SKIPPED] environ_total [23:47:26] [PASSED] environ_HOME = <0x7ffcac1b9ccb> [23:47:26] [SKIPPED] auxv_addr [23:47:26] [PASSED] auxv_AT_UID = 1000 [23:47:26] [PASSED] auxv_AT_PAGESZ = 4096 [23:47:26] ====================== [PASSED] arm64 ====================== [23:47:26] ====================== (15 subtests) ======================= [23:47:26] [PASSED] argc = 1 [23:47:26] [PASSED] argv_addr = <0x7ffdee178188> [23:47:26] [PASSED] argv_environ = <0x7ffdee178188> [23:47:26] [PASSED] argv_total = 1 [23:47:26] [PASSED] argv0_addr = <0x7ffdee178bd0> [23:47:26] [PASSED] argv0_str = <0x7ffdee178bd0> [23:47:26] [PASSED] argv0_len = 11 [23:47:26] [PASSED] environ_addr = <0x7ffdee178198> [23:47:26] [PASSED] environ_envp = <0x7ffdee178198> [23:47:26] [SKIPPED] environ_auxv [23:47:26] [SKIPPED] environ_total [23:47:26] [PASSED] environ_HOME = <0x7ffdee178ccb> [23:47:26] [SKIPPED] auxv_addr [23:47:26] [PASSED] auxv_AT_UID = 1000 [23:47:26] [PASSED] auxv_AT_PAGESZ = 4096 [23:47:26] ===================== [PASSED] x86_64 ====================== [23:47:26] ====================== (15 subtests) ======================= [23:47:26] [PASSED] argc = 1 [23:47:26] [PASSED] argv_addr = <0x7ffc16bf3628> [23:47:26] [PASSED] argv_environ = <0x7ffc16bf3628> [23:47:26] [PASSED] argv_total = 1 [23:47:26] [PASSED] argv0_addr = <0x7ffc16bf4bd0> [23:47:26] [PASSED] argv0_str = <0x7ffc16bf4bd0> [23:47:26] [PASSED] argv0_len = 11 [23:47:26] [PASSED] environ_addr = <0x7ffc16bf3638> [23:47:26] [PASSED] environ_envp = <0x7ffc16bf3638> [23:47:26] [SKIPPED] environ_auxv [23:47:26] [SKIPPED] environ_total [23:47:26] [PASSED] environ_HOME = <0x7ffc16bf4ccb> [23:47:26] [SKIPPED] auxv_addr [23:47:26] [PASSED] auxv_AT_UID = 1000 [23:47:26] [PASSED] auxv_AT_PAGESZ = 4096 [23:47:26] ===================== [PASSED] riscv64 ===================== [23:47:26] ============================================================ [23:47:26] Testing complete. Ran 45 tests: passed: 36, skipped: 9
The output of kunit.py is colored after the test results.
Not-signed-off
tools/testing/selftests/nolibc/nolibc-test.c | 121 ++++++++---------------- tools/testing/selftests/nolibc/run-all-tests.sh | 22 +++++ 2 files changed, 63 insertions(+), 80 deletions(-)
On 2023-07-20 23:29:40+0800, Zhangjin Wu wrote:
The result looks very similar and the change seems not that big, thanks.
I have a generic question: do we need to compile nolibc-test.c independently or at least let users easily compile nolibc-test.c in the other places no just in kernel source code, for example, the other libcs may want to download and compile it directly.
The functions used in this change seems not many, is it able to provide our clones for them or only provide the clones when we compile them out-of-kernel.
for example:
#ifdef NOLIBC_TEST_IN_KERNEL /* -DNOLIBC_TEST_IN_KERNEL from Makefile, for future compatibility */ #include "../kselftest.h" #else // our clones of the used functions, for standalone usage #endif
I'd rather just disable the functionality instead. Willy also asked to stick to the current formatting by default and enable KTAP with a switch. So everything will be behind generic wrappers anyways, so we can drop the KTAP functionality from those wrappers easiy.
Thomas
linux-kselftest-mirror@lists.linaro.org
-
Thomas Weißschuh -
Zhangjin Wu