Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com --- Changes v3->v4: - Rename kunit variables - Rename generator function helper macro - Add documentation for generator approach - Display test case name in case of failure along with param index Changes v2->v3: - Modifictaion of generator macro and method Changes v1->v2: - Use of a generator method to access test case parameters
include/kunit/test.h | 34 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 21 ++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ec2307ee9bb0 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,13 @@ struct kunit; * * @run_case: the function representing the actual test case. * @name: the name of the test case. + * @generate_params: the generator function for parameterized tests. + * + * The generator function is used to lazily generate a series of + * arbitrarily typed values that fit into a void*. The argument @prev + * is the previously returned value, which should be used to derive the + * next value; @prev is set to NULL on the initial generator call. + * When no more values are available, the generator must return NULL. * * A test case is a function with the signature, * ``void (*)(struct kunit *)`` @@ -141,6 +148,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name; + void* (*generate_params)(void *prev);
/* private: internal use only. */ bool success; @@ -162,6 +170,9 @@ static inline char *kunit_status_to_string(bool status) * &struct kunit_case for an example on how to use it. */ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name } +#define KUNIT_CASE_PARAM(test_name, gen_params) \ + { .run_case = test_name, .name = #test_name, \ + .generate_params = gen_params }
/** * struct kunit_suite - describes a related collection of &struct kunit_case @@ -208,6 +219,15 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch; + /* param_value points to test case parameters in parameterized tests */ + void *param_value; + /* + * param_index stores the index of the parameter in + * parameterized tests. param_index + 1 is printed + * to indicate the parameter that causes the test + * to fail in case of test failure. + */ + int param_index; /* * success starts as true, and may only be set to false during a * test case; thus, it is safe to update this across multiple @@ -1742,4 +1762,18 @@ do { \ fmt, \ ##__VA_ARGS__)
+/** + * KUNIT_ARRAY_PARAM() - Helper method for test parameter generators + * required in parameterized tests. + * @name: prefix of the name for the test parameter generator function. + * It will be suffixed by "_gen_params". + * @array: a user-supplied pointer to an array of test parameters. + */ +#define KUNIT_ARRAY_PARAM(name, array) \ + static void *name##_gen_params(void *prev) \ + { \ + typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \ + return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \ + } + #endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..8ad908b61494 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -127,6 +127,12 @@ unsigned int kunit_test_case_num(struct kunit_suite *suite, } EXPORT_SYMBOL_GPL(kunit_test_case_num);
+static void kunit_print_failed_param(struct kunit *test) +{ + kunit_err(test, "\n\tTest failed at:\n\ttest case: %s\n\tparameter: %d\n", + test->name, test->param_index + 1); +} + static void kunit_print_string_stream(struct kunit *test, struct string_stream *stream) { @@ -168,6 +174,8 @@ static void kunit_fail(struct kunit *test, struct kunit_assert *assert) assert->format(assert, stream);
kunit_print_string_stream(test, stream); + if (test->param_value) + kunit_print_failed_param(test);
WARN_ON(string_stream_destroy(stream)); } @@ -239,7 +247,18 @@ static void kunit_run_case_internal(struct kunit *test, } }
- test_case->run_case(test); + if (!test_case->generate_params) { + test_case->run_case(test); + } else { + test->param_value = test_case->generate_params(NULL); + test->param_index = 0; + + while (test->param_value) { + test_case->run_case(test); + test->param_value = test_case->generate_params(test->param_value); + test->param_index++; + } + } }
static void kunit_case_internal_cleanup(struct kunit *test)
Modify fs/ext4/inode-test.c to use the parameterized testing feature of KUnit.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com --- Changes v3->v4: - Modification based on latest implementation of KUnit parameterized testing Changes v2->v3: - Marked hardcoded test data const - Modification based on latest implementation of KUnit parameterized testing Changes v1->v2: - Modification based on latest implementation of KUnit parameterized testing
fs/ext4/inode-test.c | 314 ++++++++++++++++++++++--------------------- 1 file changed, 158 insertions(+), 156 deletions(-)
diff --git a/fs/ext4/inode-test.c b/fs/ext4/inode-test.c index d62d802c9c12..ebf1b1af4f1d 100644 --- a/fs/ext4/inode-test.c +++ b/fs/ext4/inode-test.c @@ -80,6 +80,139 @@ struct timestamp_expectation { bool lower_bound; };
+static const struct timestamp_expectation test_data[] = { + { + .test_case_name = LOWER_BOUND_NEG_NO_EXTRA_BITS_CASE, + .msb_set = true, + .lower_bound = true, + .extra_bits = 0, + .expected = {.tv_sec = -0x80000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NEG_NO_EXTRA_BITS_CASE, + .msb_set = true, + .lower_bound = false, + .extra_bits = 0, + .expected = {.tv_sec = -1LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = LOWER_BOUND_NONNEG_NO_EXTRA_BITS_CASE, + .msb_set = false, + .lower_bound = true, + .extra_bits = 0, + .expected = {0LL, 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NONNEG_NO_EXTRA_BITS_CASE, + .msb_set = false, + .lower_bound = false, + .extra_bits = 0, + .expected = {.tv_sec = 0x7fffffffLL, .tv_nsec = 0L}, + }, + + { + .test_case_name = LOWER_BOUND_NEG_LO_1_CASE, + .msb_set = true, + .lower_bound = true, + .extra_bits = 1, + .expected = {.tv_sec = 0x80000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NEG_LO_1_CASE, + .msb_set = true, + .lower_bound = false, + .extra_bits = 1, + .expected = {.tv_sec = 0xffffffffLL, .tv_nsec = 0L}, + }, + + { + .test_case_name = LOWER_BOUND_NONNEG_LO_1_CASE, + .msb_set = false, + .lower_bound = true, + .extra_bits = 1, + .expected = {.tv_sec = 0x100000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NONNEG_LO_1_CASE, + .msb_set = false, + .lower_bound = false, + .extra_bits = 1, + .expected = {.tv_sec = 0x17fffffffLL, .tv_nsec = 0L}, + }, + + { + .test_case_name = LOWER_BOUND_NEG_HI_1_CASE, + .msb_set = true, + .lower_bound = true, + .extra_bits = 2, + .expected = {.tv_sec = 0x180000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NEG_HI_1_CASE, + .msb_set = true, + .lower_bound = false, + .extra_bits = 2, + .expected = {.tv_sec = 0x1ffffffffLL, .tv_nsec = 0L}, + }, + + { + .test_case_name = LOWER_BOUND_NONNEG_HI_1_CASE, + .msb_set = false, + .lower_bound = true, + .extra_bits = 2, + .expected = {.tv_sec = 0x200000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NONNEG_HI_1_CASE, + .msb_set = false, + .lower_bound = false, + .extra_bits = 2, + .expected = {.tv_sec = 0x27fffffffLL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NONNEG_HI_1_NS_1_CASE, + .msb_set = false, + .lower_bound = false, + .extra_bits = 6, + .expected = {.tv_sec = 0x27fffffffLL, .tv_nsec = 1L}, + }, + + { + .test_case_name = LOWER_BOUND_NONNEG_HI_1_NS_MAX_CASE, + .msb_set = false, + .lower_bound = true, + .extra_bits = 0xFFFFFFFF, + .expected = {.tv_sec = 0x300000000LL, + .tv_nsec = MAX_NANOSECONDS}, + }, + + { + .test_case_name = LOWER_BOUND_NONNEG_EXTRA_BITS_1_CASE, + .msb_set = false, + .lower_bound = true, + .extra_bits = 3, + .expected = {.tv_sec = 0x300000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NONNEG_EXTRA_BITS_1_CASE, + .msb_set = false, + .lower_bound = false, + .extra_bits = 3, + .expected = {.tv_sec = 0x37fffffffLL, .tv_nsec = 0L}, + } +}; + +KUNIT_ARRAY_PARAM(ext4_inode, test_data); + static time64_t get_32bit_time(const struct timestamp_expectation * const test) { if (test->msb_set) { @@ -101,166 +234,35 @@ static time64_t get_32bit_time(const struct timestamp_expectation * const test) */ static void inode_test_xtimestamp_decoding(struct kunit *test) { - const struct timestamp_expectation test_data[] = { - { - .test_case_name = LOWER_BOUND_NEG_NO_EXTRA_BITS_CASE, - .msb_set = true, - .lower_bound = true, - .extra_bits = 0, - .expected = {.tv_sec = -0x80000000LL, .tv_nsec = 0L}, - }, - - { - .test_case_name = UPPER_BOUND_NEG_NO_EXTRA_BITS_CASE, - .msb_set = true, - .lower_bound = false, - .extra_bits = 0, - .expected = {.tv_sec = -1LL, .tv_nsec = 0L}, - }, - - { - .test_case_name = LOWER_BOUND_NONNEG_NO_EXTRA_BITS_CASE, - .msb_set = false, - .lower_bound = true, - .extra_bits = 0, - .expected = {0LL, 0L}, - }, - - { - .test_case_name = UPPER_BOUND_NONNEG_NO_EXTRA_BITS_CASE, - .msb_set = false, - .lower_bound = false, - .extra_bits = 0, - .expected = {.tv_sec = 0x7fffffffLL, .tv_nsec = 0L}, - }, - - { - .test_case_name = LOWER_BOUND_NEG_LO_1_CASE, - .msb_set = true, - .lower_bound = true, - .extra_bits = 1, - .expected = {.tv_sec = 0x80000000LL, .tv_nsec = 0L}, - }, - - { - .test_case_name = UPPER_BOUND_NEG_LO_1_CASE, - .msb_set = true, - .lower_bound = false, - .extra_bits = 1, - .expected = {.tv_sec = 0xffffffffLL, .tv_nsec = 0L}, - }, - - { - .test_case_name = LOWER_BOUND_NONNEG_LO_1_CASE, - .msb_set = false, - .lower_bound = true, - .extra_bits = 1, - .expected = {.tv_sec = 0x100000000LL, .tv_nsec = 0L}, - }, - - { - .test_case_name = UPPER_BOUND_NONNEG_LO_1_CASE, - .msb_set = false, - .lower_bound = false, - .extra_bits = 1, - .expected = {.tv_sec = 0x17fffffffLL, .tv_nsec = 0L}, - }, - - { - .test_case_name = LOWER_BOUND_NEG_HI_1_CASE, - .msb_set = true, - .lower_bound = true, - .extra_bits = 2, - .expected = {.tv_sec = 0x180000000LL, .tv_nsec = 0L}, - }, - - { - .test_case_name = UPPER_BOUND_NEG_HI_1_CASE, - .msb_set = true, - .lower_bound = false, - .extra_bits = 2, - .expected = {.tv_sec = 0x1ffffffffLL, .tv_nsec = 0L}, - }, - - { - .test_case_name = LOWER_BOUND_NONNEG_HI_1_CASE, - .msb_set = false, - .lower_bound = true, - .extra_bits = 2, - .expected = {.tv_sec = 0x200000000LL, .tv_nsec = 0L}, - }, - - { - .test_case_name = UPPER_BOUND_NONNEG_HI_1_CASE, - .msb_set = false, - .lower_bound = false, - .extra_bits = 2, - .expected = {.tv_sec = 0x27fffffffLL, .tv_nsec = 0L}, - }, - - { - .test_case_name = UPPER_BOUND_NONNEG_HI_1_NS_1_CASE, - .msb_set = false, - .lower_bound = false, - .extra_bits = 6, - .expected = {.tv_sec = 0x27fffffffLL, .tv_nsec = 1L}, - }, - - { - .test_case_name = LOWER_BOUND_NONNEG_HI_1_NS_MAX_CASE, - .msb_set = false, - .lower_bound = true, - .extra_bits = 0xFFFFFFFF, - .expected = {.tv_sec = 0x300000000LL, - .tv_nsec = MAX_NANOSECONDS}, - }, - - { - .test_case_name = LOWER_BOUND_NONNEG_EXTRA_BITS_1_CASE, - .msb_set = false, - .lower_bound = true, - .extra_bits = 3, - .expected = {.tv_sec = 0x300000000LL, .tv_nsec = 0L}, - }, - - { - .test_case_name = UPPER_BOUND_NONNEG_EXTRA_BITS_1_CASE, - .msb_set = false, - .lower_bound = false, - .extra_bits = 3, - .expected = {.tv_sec = 0x37fffffffLL, .tv_nsec = 0L}, - } - }; - struct timespec64 timestamp; - int i; - - for (i = 0; i < ARRAY_SIZE(test_data); ++i) { - timestamp.tv_sec = get_32bit_time(&test_data[i]); - ext4_decode_extra_time(×tamp, - cpu_to_le32(test_data[i].extra_bits)); - - KUNIT_EXPECT_EQ_MSG(test, - test_data[i].expected.tv_sec, - timestamp.tv_sec, - CASE_NAME_FORMAT, - test_data[i].test_case_name, - test_data[i].msb_set, - test_data[i].lower_bound, - test_data[i].extra_bits); - KUNIT_EXPECT_EQ_MSG(test, - test_data[i].expected.tv_nsec, - timestamp.tv_nsec, - CASE_NAME_FORMAT, - test_data[i].test_case_name, - test_data[i].msb_set, - test_data[i].lower_bound, - test_data[i].extra_bits); - } + + struct timestamp_expectation *test_param = + (struct timestamp_expectation *)(test->param_value); + + timestamp.tv_sec = get_32bit_time(test_param); + ext4_decode_extra_time(×tamp, + cpu_to_le32(test_param->extra_bits)); + + KUNIT_EXPECT_EQ_MSG(test, + test_param->expected.tv_sec, + timestamp.tv_sec, + CASE_NAME_FORMAT, + test_param->test_case_name, + test_param->msb_set, + test_param->lower_bound, + test_param->extra_bits); + KUNIT_EXPECT_EQ_MSG(test, + test_param->expected.tv_nsec, + timestamp.tv_nsec, + CASE_NAME_FORMAT, + test_param->test_case_name, + test_param->msb_set, + test_param->lower_bound, + test_param->extra_bits); }
static struct kunit_case ext4_inode_test_cases[] = { - KUNIT_CASE(inode_test_xtimestamp_decoding), + KUNIT_CASE_PARAM(inode_test_xtimestamp_decoding, ext4_inode_gen_params), {} };
Hi Arpitha,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on ext4/dev] [also build test WARNING on linus/master v5.10-rc1 next-20201027] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Arpitha-Raghunandan/kunit-Support-f... base: https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git dev config: mips-randconfig-r016-20201027 (attached as .config) compiler: mipsel-linux-gcc (GCC) 9.3.0 reproduce (this is a W=1 build): wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross # https://github.com/0day-ci/linux/commit/2de1e52708cd83d1dc4c718876683f680904... git remote add linux-review https://github.com/0day-ci/linux git fetch --no-tags linux-review Arpitha-Raghunandan/kunit-Support-for-Parameterized-Testing/20201028-015018 git checkout 2de1e52708cd83d1dc4c718876683f6809045a98 # save the attached .config to linux build tree COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=mips
If you fix the issue, kindly add following tag as appropriate Reported-by: kernel test robot lkp@intel.com
All warnings (new ones prefixed by >>):
In file included from fs/ext4/inode-test.c:7: fs/ext4/inode-test.c: In function 'ext4_inode_gen_params':
include/kunit/test.h:1735:58: warning: return discards 'const' qualifier from pointer target type [-Wdiscarded-qualifiers]
1735 | return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \ fs/ext4/inode-test.c:214:1: note: in expansion of macro 'KUNIT_ARRAY_PARAM' 214 | KUNIT_ARRAY_PARAM(ext4_inode, test_data); | ^~~~~~~~~~~~~~~~~
vim +/const +1735 include/kunit/test.h
73cda7bb8bfb1d Brendan Higgins 2019-09-23 1154 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1155 #define KUNIT_PTR_NOT_ERR_OR_NULL_ASSERTION(test, assert_type, ptr) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1156 KUNIT_PTR_NOT_ERR_OR_NULL_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1157 assert_type, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1158 ptr, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1159 NULL) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1160 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1161 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1162 * KUNIT_EXPECT_TRUE() - Causes a test failure when the expression is not true. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1163 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1164 * @condition: an arbitrary boolean expression. The test fails when this does 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1165 * not evaluate to true. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1166 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1167 * This and expectations of the form `KUNIT_EXPECT_*` will cause the test case 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1168 * to fail when the specified condition is not met; however, it will not prevent 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1169 * the test case from continuing to run; this is otherwise known as an 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1170 * *expectation failure*. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1171 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1172 #define KUNIT_EXPECT_TRUE(test, condition) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1173 KUNIT_TRUE_ASSERTION(test, KUNIT_EXPECTATION, condition) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1174 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1175 #define KUNIT_EXPECT_TRUE_MSG(test, condition, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1176 KUNIT_TRUE_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1177 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1178 condition, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1179 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1180 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1181 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1182 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1183 * KUNIT_EXPECT_FALSE() - Makes a test failure when the expression is not false. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1184 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1185 * @condition: an arbitrary boolean expression. The test fails when this does 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1186 * not evaluate to false. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1187 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1188 * Sets an expectation that @condition evaluates to false. See 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1189 * KUNIT_EXPECT_TRUE() for more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1190 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1191 #define KUNIT_EXPECT_FALSE(test, condition) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1192 KUNIT_FALSE_ASSERTION(test, KUNIT_EXPECTATION, condition) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1193 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1194 #define KUNIT_EXPECT_FALSE_MSG(test, condition, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1195 KUNIT_FALSE_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1196 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1197 condition, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1198 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1199 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1200 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1201 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1202 * KUNIT_EXPECT_EQ() - Sets an expectation that @left and @right are equal. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1203 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1204 * @left: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1205 * @right: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1206 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1207 * Sets an expectation that the values that @left and @right evaluate to are 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1208 * equal. This is semantically equivalent to 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1209 * KUNIT_EXPECT_TRUE(@test, (@left) == (@right)). See KUNIT_EXPECT_TRUE() for 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1210 * more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1211 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1212 #define KUNIT_EXPECT_EQ(test, left, right) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1213 KUNIT_BINARY_EQ_ASSERTION(test, KUNIT_EXPECTATION, left, right) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1214 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1215 #define KUNIT_EXPECT_EQ_MSG(test, left, right, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1216 KUNIT_BINARY_EQ_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1217 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1218 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1219 right, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1220 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1221 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1222 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1223 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1224 * KUNIT_EXPECT_PTR_EQ() - Expects that pointers @left and @right are equal. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1225 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1226 * @left: an arbitrary expression that evaluates to a pointer. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1227 * @right: an arbitrary expression that evaluates to a pointer. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1228 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1229 * Sets an expectation that the values that @left and @right evaluate to are 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1230 * equal. This is semantically equivalent to 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1231 * KUNIT_EXPECT_TRUE(@test, (@left) == (@right)). See KUNIT_EXPECT_TRUE() for 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1232 * more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1233 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1234 #define KUNIT_EXPECT_PTR_EQ(test, left, right) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1235 KUNIT_BINARY_PTR_EQ_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1236 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1237 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1238 right) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1239 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1240 #define KUNIT_EXPECT_PTR_EQ_MSG(test, left, right, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1241 KUNIT_BINARY_PTR_EQ_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1242 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1243 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1244 right, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1245 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1246 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1247 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1248 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1249 * KUNIT_EXPECT_NE() - An expectation that @left and @right are not equal. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1250 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1251 * @left: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1252 * @right: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1253 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1254 * Sets an expectation that the values that @left and @right evaluate to are not 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1255 * equal. This is semantically equivalent to 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1256 * KUNIT_EXPECT_TRUE(@test, (@left) != (@right)). See KUNIT_EXPECT_TRUE() for 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1257 * more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1258 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1259 #define KUNIT_EXPECT_NE(test, left, right) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1260 KUNIT_BINARY_NE_ASSERTION(test, KUNIT_EXPECTATION, left, right) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1261 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1262 #define KUNIT_EXPECT_NE_MSG(test, left, right, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1263 KUNIT_BINARY_NE_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1264 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1265 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1266 right, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1267 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1268 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1269 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1270 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1271 * KUNIT_EXPECT_PTR_NE() - Expects that pointers @left and @right are not equal. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1272 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1273 * @left: an arbitrary expression that evaluates to a pointer. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1274 * @right: an arbitrary expression that evaluates to a pointer. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1275 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1276 * Sets an expectation that the values that @left and @right evaluate to are not 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1277 * equal. This is semantically equivalent to 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1278 * KUNIT_EXPECT_TRUE(@test, (@left) != (@right)). See KUNIT_EXPECT_TRUE() for 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1279 * more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1280 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1281 #define KUNIT_EXPECT_PTR_NE(test, left, right) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1282 KUNIT_BINARY_PTR_NE_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1283 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1284 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1285 right) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1286 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1287 #define KUNIT_EXPECT_PTR_NE_MSG(test, left, right, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1288 KUNIT_BINARY_PTR_NE_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1289 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1290 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1291 right, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1292 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1293 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1294 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1295 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1296 * KUNIT_EXPECT_LT() - An expectation that @left is less than @right. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1297 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1298 * @left: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1299 * @right: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1300 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1301 * Sets an expectation that the value that @left evaluates to is less than the 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1302 * value that @right evaluates to. This is semantically equivalent to 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1303 * KUNIT_EXPECT_TRUE(@test, (@left) < (@right)). See KUNIT_EXPECT_TRUE() for 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1304 * more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1305 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1306 #define KUNIT_EXPECT_LT(test, left, right) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1307 KUNIT_BINARY_LT_ASSERTION(test, KUNIT_EXPECTATION, left, right) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1308 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1309 #define KUNIT_EXPECT_LT_MSG(test, left, right, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1310 KUNIT_BINARY_LT_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1311 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1312 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1313 right, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1314 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1315 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1316 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1317 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1318 * KUNIT_EXPECT_LE() - Expects that @left is less than or equal to @right. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1319 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1320 * @left: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1321 * @right: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1322 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1323 * Sets an expectation that the value that @left evaluates to is less than or 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1324 * equal to the value that @right evaluates to. Semantically this is equivalent 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1325 * to KUNIT_EXPECT_TRUE(@test, (@left) <= (@right)). See KUNIT_EXPECT_TRUE() for 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1326 * more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1327 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1328 #define KUNIT_EXPECT_LE(test, left, right) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1329 KUNIT_BINARY_LE_ASSERTION(test, KUNIT_EXPECTATION, left, right) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1330 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1331 #define KUNIT_EXPECT_LE_MSG(test, left, right, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1332 KUNIT_BINARY_LE_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1333 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1334 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1335 right, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1336 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1337 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1338 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1339 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1340 * KUNIT_EXPECT_GT() - An expectation that @left is greater than @right. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1341 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1342 * @left: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1343 * @right: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1344 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1345 * Sets an expectation that the value that @left evaluates to is greater than 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1346 * the value that @right evaluates to. This is semantically equivalent to 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1347 * KUNIT_EXPECT_TRUE(@test, (@left) > (@right)). See KUNIT_EXPECT_TRUE() for 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1348 * more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1349 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1350 #define KUNIT_EXPECT_GT(test, left, right) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1351 KUNIT_BINARY_GT_ASSERTION(test, KUNIT_EXPECTATION, left, right) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1352 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1353 #define KUNIT_EXPECT_GT_MSG(test, left, right, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1354 KUNIT_BINARY_GT_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1355 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1356 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1357 right, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1358 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1359 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1360 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1361 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1362 * KUNIT_EXPECT_GE() - Expects that @left is greater than or equal to @right. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1363 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1364 * @left: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1365 * @right: an arbitrary expression that evaluates to a primitive C type. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1366 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1367 * Sets an expectation that the value that @left evaluates to is greater than 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1368 * the value that @right evaluates to. This is semantically equivalent to 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1369 * KUNIT_EXPECT_TRUE(@test, (@left) >= (@right)). See KUNIT_EXPECT_TRUE() for 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1370 * more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1371 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1372 #define KUNIT_EXPECT_GE(test, left, right) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1373 KUNIT_BINARY_GE_ASSERTION(test, KUNIT_EXPECTATION, left, right) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1374 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1375 #define KUNIT_EXPECT_GE_MSG(test, left, right, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1376 KUNIT_BINARY_GE_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1377 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1378 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1379 right, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1380 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1381 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1382 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1383 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1384 * KUNIT_EXPECT_STREQ() - Expects that strings @left and @right are equal. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1385 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1386 * @left: an arbitrary expression that evaluates to a null terminated string. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1387 * @right: an arbitrary expression that evaluates to a null terminated string. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1388 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1389 * Sets an expectation that the values that @left and @right evaluate to are 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1390 * equal. This is semantically equivalent to 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1391 * KUNIT_EXPECT_TRUE(@test, !strcmp((@left), (@right))). See KUNIT_EXPECT_TRUE() 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1392 * for more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1393 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1394 #define KUNIT_EXPECT_STREQ(test, left, right) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1395 KUNIT_BINARY_STR_EQ_ASSERTION(test, KUNIT_EXPECTATION, left, right) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1396 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1397 #define KUNIT_EXPECT_STREQ_MSG(test, left, right, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1398 KUNIT_BINARY_STR_EQ_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1399 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1400 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1401 right, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1402 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1403 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1404 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1405 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1406 * KUNIT_EXPECT_STRNEQ() - Expects that strings @left and @right are not equal. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1407 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1408 * @left: an arbitrary expression that evaluates to a null terminated string. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1409 * @right: an arbitrary expression that evaluates to a null terminated string. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1410 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1411 * Sets an expectation that the values that @left and @right evaluate to are 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1412 * not equal. This is semantically equivalent to 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1413 * KUNIT_EXPECT_TRUE(@test, strcmp((@left), (@right))). See KUNIT_EXPECT_TRUE() 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1414 * for more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1415 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1416 #define KUNIT_EXPECT_STRNEQ(test, left, right) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1417 KUNIT_BINARY_STR_NE_ASSERTION(test, KUNIT_EXPECTATION, left, right) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1418 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1419 #define KUNIT_EXPECT_STRNEQ_MSG(test, left, right, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1420 KUNIT_BINARY_STR_NE_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1421 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1422 left, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1423 right, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1424 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1425 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1426 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1427 /** 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1428 * KUNIT_EXPECT_NOT_ERR_OR_NULL() - Expects that @ptr is not null and not err. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1429 * @test: The test context object. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1430 * @ptr: an arbitrary pointer. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1431 * 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1432 * Sets an expectation that the value that @ptr evaluates to is not null and not 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1433 * an errno stored in a pointer. This is semantically equivalent to 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1434 * KUNIT_EXPECT_TRUE(@test, !IS_ERR_OR_NULL(@ptr)). See KUNIT_EXPECT_TRUE() for 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1435 * more information. 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1436 */ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1437 #define KUNIT_EXPECT_NOT_ERR_OR_NULL(test, ptr) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1438 KUNIT_PTR_NOT_ERR_OR_NULL_ASSERTION(test, KUNIT_EXPECTATION, ptr) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1439 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1440 #define KUNIT_EXPECT_NOT_ERR_OR_NULL_MSG(test, ptr, fmt, ...) \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1441 KUNIT_PTR_NOT_ERR_OR_NULL_MSG_ASSERTION(test, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1442 KUNIT_EXPECTATION, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1443 ptr, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1444 fmt, \ 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1445 ##__VA_ARGS__) 73cda7bb8bfb1d Brendan Higgins 2019-09-23 1446 e4aea8f8532b55 Brendan Higgins 2019-09-23 1447 #define KUNIT_ASSERT_FAILURE(test, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1448 KUNIT_FAIL_ASSERTION(test, KUNIT_ASSERTION, fmt, ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1449 e4aea8f8532b55 Brendan Higgins 2019-09-23 1450 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1451 * KUNIT_ASSERT_TRUE() - Sets an assertion that @condition is true. e4aea8f8532b55 Brendan Higgins 2019-09-23 1452 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1453 * @condition: an arbitrary boolean expression. The test fails and aborts when e4aea8f8532b55 Brendan Higgins 2019-09-23 1454 * this does not evaluate to true. e4aea8f8532b55 Brendan Higgins 2019-09-23 1455 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1456 * This and assertions of the form `KUNIT_ASSERT_*` will cause the test case to e4aea8f8532b55 Brendan Higgins 2019-09-23 1457 * fail *and immediately abort* when the specified condition is not met. Unlike e4aea8f8532b55 Brendan Higgins 2019-09-23 1458 * an expectation failure, it will prevent the test case from continuing to run; e4aea8f8532b55 Brendan Higgins 2019-09-23 1459 * this is otherwise known as an *assertion failure*. e4aea8f8532b55 Brendan Higgins 2019-09-23 1460 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1461 #define KUNIT_ASSERT_TRUE(test, condition) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1462 KUNIT_TRUE_ASSERTION(test, KUNIT_ASSERTION, condition) e4aea8f8532b55 Brendan Higgins 2019-09-23 1463 e4aea8f8532b55 Brendan Higgins 2019-09-23 1464 #define KUNIT_ASSERT_TRUE_MSG(test, condition, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1465 KUNIT_TRUE_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1466 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1467 condition, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1468 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1469 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1470 e4aea8f8532b55 Brendan Higgins 2019-09-23 1471 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1472 * KUNIT_ASSERT_FALSE() - Sets an assertion that @condition is false. e4aea8f8532b55 Brendan Higgins 2019-09-23 1473 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1474 * @condition: an arbitrary boolean expression. e4aea8f8532b55 Brendan Higgins 2019-09-23 1475 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1476 * Sets an assertion that the value that @condition evaluates to is false. This e4aea8f8532b55 Brendan Higgins 2019-09-23 1477 * is the same as KUNIT_EXPECT_FALSE(), except it causes an assertion failure e4aea8f8532b55 Brendan Higgins 2019-09-23 1478 * (see KUNIT_ASSERT_TRUE()) when the assertion is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1479 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1480 #define KUNIT_ASSERT_FALSE(test, condition) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1481 KUNIT_FALSE_ASSERTION(test, KUNIT_ASSERTION, condition) e4aea8f8532b55 Brendan Higgins 2019-09-23 1482 e4aea8f8532b55 Brendan Higgins 2019-09-23 1483 #define KUNIT_ASSERT_FALSE_MSG(test, condition, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1484 KUNIT_FALSE_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1485 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1486 condition, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1487 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1488 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1489 e4aea8f8532b55 Brendan Higgins 2019-09-23 1490 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1491 * KUNIT_ASSERT_EQ() - Sets an assertion that @left and @right are equal. e4aea8f8532b55 Brendan Higgins 2019-09-23 1492 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1493 * @left: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1494 * @right: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1495 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1496 * Sets an assertion that the values that @left and @right evaluate to are e4aea8f8532b55 Brendan Higgins 2019-09-23 1497 * equal. This is the same as KUNIT_EXPECT_EQ(), except it causes an assertion e4aea8f8532b55 Brendan Higgins 2019-09-23 1498 * failure (see KUNIT_ASSERT_TRUE()) when the assertion is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1499 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1500 #define KUNIT_ASSERT_EQ(test, left, right) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1501 KUNIT_BINARY_EQ_ASSERTION(test, KUNIT_ASSERTION, left, right) e4aea8f8532b55 Brendan Higgins 2019-09-23 1502 e4aea8f8532b55 Brendan Higgins 2019-09-23 1503 #define KUNIT_ASSERT_EQ_MSG(test, left, right, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1504 KUNIT_BINARY_EQ_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1505 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1506 left, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1507 right, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1508 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1509 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1510 e4aea8f8532b55 Brendan Higgins 2019-09-23 1511 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1512 * KUNIT_ASSERT_PTR_EQ() - Asserts that pointers @left and @right are equal. e4aea8f8532b55 Brendan Higgins 2019-09-23 1513 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1514 * @left: an arbitrary expression that evaluates to a pointer. e4aea8f8532b55 Brendan Higgins 2019-09-23 1515 * @right: an arbitrary expression that evaluates to a pointer. e4aea8f8532b55 Brendan Higgins 2019-09-23 1516 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1517 * Sets an assertion that the values that @left and @right evaluate to are e4aea8f8532b55 Brendan Higgins 2019-09-23 1518 * equal. This is the same as KUNIT_EXPECT_EQ(), except it causes an assertion e4aea8f8532b55 Brendan Higgins 2019-09-23 1519 * failure (see KUNIT_ASSERT_TRUE()) when the assertion is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1520 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1521 #define KUNIT_ASSERT_PTR_EQ(test, left, right) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1522 KUNIT_BINARY_PTR_EQ_ASSERTION(test, KUNIT_ASSERTION, left, right) e4aea8f8532b55 Brendan Higgins 2019-09-23 1523 e4aea8f8532b55 Brendan Higgins 2019-09-23 1524 #define KUNIT_ASSERT_PTR_EQ_MSG(test, left, right, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1525 KUNIT_BINARY_PTR_EQ_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1526 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1527 left, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1528 right, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1529 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1530 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1531 e4aea8f8532b55 Brendan Higgins 2019-09-23 1532 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1533 * KUNIT_ASSERT_NE() - An assertion that @left and @right are not equal. e4aea8f8532b55 Brendan Higgins 2019-09-23 1534 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1535 * @left: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1536 * @right: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1537 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1538 * Sets an assertion that the values that @left and @right evaluate to are not e4aea8f8532b55 Brendan Higgins 2019-09-23 1539 * equal. This is the same as KUNIT_EXPECT_NE(), except it causes an assertion e4aea8f8532b55 Brendan Higgins 2019-09-23 1540 * failure (see KUNIT_ASSERT_TRUE()) when the assertion is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1541 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1542 #define KUNIT_ASSERT_NE(test, left, right) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1543 KUNIT_BINARY_NE_ASSERTION(test, KUNIT_ASSERTION, left, right) e4aea8f8532b55 Brendan Higgins 2019-09-23 1544 e4aea8f8532b55 Brendan Higgins 2019-09-23 1545 #define KUNIT_ASSERT_NE_MSG(test, left, right, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1546 KUNIT_BINARY_NE_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1547 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1548 left, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1549 right, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1550 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1551 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1552 e4aea8f8532b55 Brendan Higgins 2019-09-23 1553 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1554 * KUNIT_ASSERT_PTR_NE() - Asserts that pointers @left and @right are not equal. e4aea8f8532b55 Brendan Higgins 2019-09-23 1555 * KUNIT_ASSERT_PTR_EQ() - Asserts that pointers @left and @right are equal. e4aea8f8532b55 Brendan Higgins 2019-09-23 1556 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1557 * @left: an arbitrary expression that evaluates to a pointer. e4aea8f8532b55 Brendan Higgins 2019-09-23 1558 * @right: an arbitrary expression that evaluates to a pointer. e4aea8f8532b55 Brendan Higgins 2019-09-23 1559 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1560 * Sets an assertion that the values that @left and @right evaluate to are not e4aea8f8532b55 Brendan Higgins 2019-09-23 1561 * equal. This is the same as KUNIT_EXPECT_NE(), except it causes an assertion e4aea8f8532b55 Brendan Higgins 2019-09-23 1562 * failure (see KUNIT_ASSERT_TRUE()) when the assertion is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1563 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1564 #define KUNIT_ASSERT_PTR_NE(test, left, right) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1565 KUNIT_BINARY_PTR_NE_ASSERTION(test, KUNIT_ASSERTION, left, right) e4aea8f8532b55 Brendan Higgins 2019-09-23 1566 e4aea8f8532b55 Brendan Higgins 2019-09-23 1567 #define KUNIT_ASSERT_PTR_NE_MSG(test, left, right, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1568 KUNIT_BINARY_PTR_NE_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1569 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1570 left, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1571 right, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1572 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1573 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1574 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1575 * KUNIT_ASSERT_LT() - An assertion that @left is less than @right. e4aea8f8532b55 Brendan Higgins 2019-09-23 1576 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1577 * @left: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1578 * @right: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1579 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1580 * Sets an assertion that the value that @left evaluates to is less than the e4aea8f8532b55 Brendan Higgins 2019-09-23 1581 * value that @right evaluates to. This is the same as KUNIT_EXPECT_LT(), except e4aea8f8532b55 Brendan Higgins 2019-09-23 1582 * it causes an assertion failure (see KUNIT_ASSERT_TRUE()) when the assertion e4aea8f8532b55 Brendan Higgins 2019-09-23 1583 * is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1584 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1585 #define KUNIT_ASSERT_LT(test, left, right) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1586 KUNIT_BINARY_LT_ASSERTION(test, KUNIT_ASSERTION, left, right) e4aea8f8532b55 Brendan Higgins 2019-09-23 1587 e4aea8f8532b55 Brendan Higgins 2019-09-23 1588 #define KUNIT_ASSERT_LT_MSG(test, left, right, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1589 KUNIT_BINARY_LT_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1590 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1591 left, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1592 right, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1593 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1594 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1595 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1596 * KUNIT_ASSERT_LE() - An assertion that @left is less than or equal to @right. e4aea8f8532b55 Brendan Higgins 2019-09-23 1597 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1598 * @left: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1599 * @right: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1600 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1601 * Sets an assertion that the value that @left evaluates to is less than or e4aea8f8532b55 Brendan Higgins 2019-09-23 1602 * equal to the value that @right evaluates to. This is the same as e4aea8f8532b55 Brendan Higgins 2019-09-23 1603 * KUNIT_EXPECT_LE(), except it causes an assertion failure (see e4aea8f8532b55 Brendan Higgins 2019-09-23 1604 * KUNIT_ASSERT_TRUE()) when the assertion is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1605 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1606 #define KUNIT_ASSERT_LE(test, left, right) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1607 KUNIT_BINARY_LE_ASSERTION(test, KUNIT_ASSERTION, left, right) e4aea8f8532b55 Brendan Higgins 2019-09-23 1608 e4aea8f8532b55 Brendan Higgins 2019-09-23 1609 #define KUNIT_ASSERT_LE_MSG(test, left, right, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1610 KUNIT_BINARY_LE_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1611 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1612 left, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1613 right, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1614 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1615 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1616 e4aea8f8532b55 Brendan Higgins 2019-09-23 1617 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1618 * KUNIT_ASSERT_GT() - An assertion that @left is greater than @right. e4aea8f8532b55 Brendan Higgins 2019-09-23 1619 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1620 * @left: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1621 * @right: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1622 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1623 * Sets an assertion that the value that @left evaluates to is greater than the e4aea8f8532b55 Brendan Higgins 2019-09-23 1624 * value that @right evaluates to. This is the same as KUNIT_EXPECT_GT(), except e4aea8f8532b55 Brendan Higgins 2019-09-23 1625 * it causes an assertion failure (see KUNIT_ASSERT_TRUE()) when the assertion e4aea8f8532b55 Brendan Higgins 2019-09-23 1626 * is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1627 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1628 #define KUNIT_ASSERT_GT(test, left, right) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1629 KUNIT_BINARY_GT_ASSERTION(test, KUNIT_ASSERTION, left, right) e4aea8f8532b55 Brendan Higgins 2019-09-23 1630 e4aea8f8532b55 Brendan Higgins 2019-09-23 1631 #define KUNIT_ASSERT_GT_MSG(test, left, right, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1632 KUNIT_BINARY_GT_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1633 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1634 left, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1635 right, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1636 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1637 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1638 e4aea8f8532b55 Brendan Higgins 2019-09-23 1639 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1640 * KUNIT_ASSERT_GE() - Assertion that @left is greater than or equal to @right. e4aea8f8532b55 Brendan Higgins 2019-09-23 1641 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1642 * @left: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1643 * @right: an arbitrary expression that evaluates to a primitive C type. e4aea8f8532b55 Brendan Higgins 2019-09-23 1644 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1645 * Sets an assertion that the value that @left evaluates to is greater than the e4aea8f8532b55 Brendan Higgins 2019-09-23 1646 * value that @right evaluates to. This is the same as KUNIT_EXPECT_GE(), except e4aea8f8532b55 Brendan Higgins 2019-09-23 1647 * it causes an assertion failure (see KUNIT_ASSERT_TRUE()) when the assertion e4aea8f8532b55 Brendan Higgins 2019-09-23 1648 * is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1649 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1650 #define KUNIT_ASSERT_GE(test, left, right) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1651 KUNIT_BINARY_GE_ASSERTION(test, KUNIT_ASSERTION, left, right) e4aea8f8532b55 Brendan Higgins 2019-09-23 1652 e4aea8f8532b55 Brendan Higgins 2019-09-23 1653 #define KUNIT_ASSERT_GE_MSG(test, left, right, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1654 KUNIT_BINARY_GE_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1655 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1656 left, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1657 right, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1658 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1659 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1660 e4aea8f8532b55 Brendan Higgins 2019-09-23 1661 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1662 * KUNIT_ASSERT_STREQ() - An assertion that strings @left and @right are equal. e4aea8f8532b55 Brendan Higgins 2019-09-23 1663 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1664 * @left: an arbitrary expression that evaluates to a null terminated string. e4aea8f8532b55 Brendan Higgins 2019-09-23 1665 * @right: an arbitrary expression that evaluates to a null terminated string. e4aea8f8532b55 Brendan Higgins 2019-09-23 1666 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1667 * Sets an assertion that the values that @left and @right evaluate to are e4aea8f8532b55 Brendan Higgins 2019-09-23 1668 * equal. This is the same as KUNIT_EXPECT_STREQ(), except it causes an e4aea8f8532b55 Brendan Higgins 2019-09-23 1669 * assertion failure (see KUNIT_ASSERT_TRUE()) when the assertion is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1670 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1671 #define KUNIT_ASSERT_STREQ(test, left, right) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1672 KUNIT_BINARY_STR_EQ_ASSERTION(test, KUNIT_ASSERTION, left, right) e4aea8f8532b55 Brendan Higgins 2019-09-23 1673 e4aea8f8532b55 Brendan Higgins 2019-09-23 1674 #define KUNIT_ASSERT_STREQ_MSG(test, left, right, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1675 KUNIT_BINARY_STR_EQ_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1676 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1677 left, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1678 right, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1679 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1680 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1681 e4aea8f8532b55 Brendan Higgins 2019-09-23 1682 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1683 * KUNIT_ASSERT_STRNEQ() - Expects that strings @left and @right are not equal. e4aea8f8532b55 Brendan Higgins 2019-09-23 1684 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1685 * @left: an arbitrary expression that evaluates to a null terminated string. e4aea8f8532b55 Brendan Higgins 2019-09-23 1686 * @right: an arbitrary expression that evaluates to a null terminated string. e4aea8f8532b55 Brendan Higgins 2019-09-23 1687 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1688 * Sets an expectation that the values that @left and @right evaluate to are e4aea8f8532b55 Brendan Higgins 2019-09-23 1689 * not equal. This is semantically equivalent to e4aea8f8532b55 Brendan Higgins 2019-09-23 1690 * KUNIT_ASSERT_TRUE(@test, strcmp((@left), (@right))). See KUNIT_ASSERT_TRUE() e4aea8f8532b55 Brendan Higgins 2019-09-23 1691 * for more information. e4aea8f8532b55 Brendan Higgins 2019-09-23 1692 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1693 #define KUNIT_ASSERT_STRNEQ(test, left, right) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1694 KUNIT_BINARY_STR_NE_ASSERTION(test, KUNIT_ASSERTION, left, right) e4aea8f8532b55 Brendan Higgins 2019-09-23 1695 e4aea8f8532b55 Brendan Higgins 2019-09-23 1696 #define KUNIT_ASSERT_STRNEQ_MSG(test, left, right, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1697 KUNIT_BINARY_STR_NE_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1698 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1699 left, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1700 right, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1701 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1702 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1703 e4aea8f8532b55 Brendan Higgins 2019-09-23 1704 /** e4aea8f8532b55 Brendan Higgins 2019-09-23 1705 * KUNIT_ASSERT_NOT_ERR_OR_NULL() - Assertion that @ptr is not null and not err. e4aea8f8532b55 Brendan Higgins 2019-09-23 1706 * @test: The test context object. e4aea8f8532b55 Brendan Higgins 2019-09-23 1707 * @ptr: an arbitrary pointer. e4aea8f8532b55 Brendan Higgins 2019-09-23 1708 * e4aea8f8532b55 Brendan Higgins 2019-09-23 1709 * Sets an assertion that the value that @ptr evaluates to is not null and not e4aea8f8532b55 Brendan Higgins 2019-09-23 1710 * an errno stored in a pointer. This is the same as e4aea8f8532b55 Brendan Higgins 2019-09-23 1711 * KUNIT_EXPECT_NOT_ERR_OR_NULL(), except it causes an assertion failure (see e4aea8f8532b55 Brendan Higgins 2019-09-23 1712 * KUNIT_ASSERT_TRUE()) when the assertion is not met. e4aea8f8532b55 Brendan Higgins 2019-09-23 1713 */ e4aea8f8532b55 Brendan Higgins 2019-09-23 1714 #define KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1715 KUNIT_PTR_NOT_ERR_OR_NULL_ASSERTION(test, KUNIT_ASSERTION, ptr) e4aea8f8532b55 Brendan Higgins 2019-09-23 1716 e4aea8f8532b55 Brendan Higgins 2019-09-23 1717 #define KUNIT_ASSERT_NOT_ERR_OR_NULL_MSG(test, ptr, fmt, ...) \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1718 KUNIT_PTR_NOT_ERR_OR_NULL_MSG_ASSERTION(test, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1719 KUNIT_ASSERTION, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1720 ptr, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1721 fmt, \ e4aea8f8532b55 Brendan Higgins 2019-09-23 1722 ##__VA_ARGS__) e4aea8f8532b55 Brendan Higgins 2019-09-23 1723 ae68283f3e666a Arpitha Raghunandan 2020-10-27 1724 /** ae68283f3e666a Arpitha Raghunandan 2020-10-27 1725 * KUNIT_ARRAY_PARAM() - Helper method for test parameter generators ae68283f3e666a Arpitha Raghunandan 2020-10-27 1726 * required in parameterized tests. ae68283f3e666a Arpitha Raghunandan 2020-10-27 1727 * @name: prefix of the name for the test parameter generator function. ae68283f3e666a Arpitha Raghunandan 2020-10-27 1728 * It will be suffixed by "_gen_params". ae68283f3e666a Arpitha Raghunandan 2020-10-27 1729 * @array: a user-supplied pointer to an array of test parameters. ae68283f3e666a Arpitha Raghunandan 2020-10-27 1730 */ ae68283f3e666a Arpitha Raghunandan 2020-10-27 1731 #define KUNIT_ARRAY_PARAM(name, array) \ ae68283f3e666a Arpitha Raghunandan 2020-10-27 1732 static void *name##_gen_params(void *prev) \ ae68283f3e666a Arpitha Raghunandan 2020-10-27 1733 { \ ae68283f3e666a Arpitha Raghunandan 2020-10-27 1734 typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \ ae68283f3e666a Arpitha Raghunandan 2020-10-27 @1735 return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \ ae68283f3e666a Arpitha Raghunandan 2020-10-27 1736 } ae68283f3e666a Arpitha Raghunandan 2020-10-27 1737
--- 0-DAY CI Kernel Test Service, Intel Corporation https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
On Wed, 28 Oct 2020 at 00:50, kernel test robot lkp@intel.com wrote:
Hi Arpitha,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on ext4/dev] [also build test WARNING on linus/master v5.10-rc1 next-20201027] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Arpitha-Raghunandan/kunit-Support-f... base: https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git dev config: mips-randconfig-r016-20201027 (attached as .config) compiler: mipsel-linux-gcc (GCC) 9.3.0 reproduce (this is a W=1 build): wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross # https://github.com/0day-ci/linux/commit/2de1e52708cd83d1dc4c718876683f680904... git remote add linux-review https://github.com/0day-ci/linux git fetch --no-tags linux-review Arpitha-Raghunandan/kunit-Support-for-Parameterized-Testing/20201028-015018 git checkout 2de1e52708cd83d1dc4c718876683f6809045a98 # save the attached .config to linux build tree COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=mips
If you fix the issue, kindly add following tag as appropriate Reported-by: kernel test robot lkp@intel.com
All warnings (new ones prefixed by >>):
In file included from fs/ext4/inode-test.c:7: fs/ext4/inode-test.c: In function 'ext4_inode_gen_params':
include/kunit/test.h:1735:58: warning: return discards 'const' qualifier from pointer target type [-Wdiscarded-qualifiers]
1735 | return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \
fs/ext4/inode-test.c:214:1: note: in expansion of macro 'KUNIT_ARRAY_PARAM' 214 | KUNIT_ARRAY_PARAM(ext4_inode, test_data); | ^~~~~~~~~~~~~~~~~
So this means we probably want to make the param_value, and the return and prev types of the generator "const void*".
Thanks, -- Marco
On 28/10/20 2:00 pm, Marco Elver wrote:
On Wed, 28 Oct 2020 at 00:50, kernel test robot lkp@intel.com wrote:
Hi Arpitha,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on ext4/dev] [also build test WARNING on linus/master v5.10-rc1 next-20201027] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Arpitha-Raghunandan/kunit-Support-f... base: https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git dev config: mips-randconfig-r016-20201027 (attached as .config) compiler: mipsel-linux-gcc (GCC) 9.3.0 reproduce (this is a W=1 build): wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross # https://github.com/0day-ci/linux/commit/2de1e52708cd83d1dc4c718876683f680904... git remote add linux-review https://github.com/0day-ci/linux git fetch --no-tags linux-review Arpitha-Raghunandan/kunit-Support-for-Parameterized-Testing/20201028-015018 git checkout 2de1e52708cd83d1dc4c718876683f6809045a98 # save the attached .config to linux build tree COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=mips
If you fix the issue, kindly add following tag as appropriate Reported-by: kernel test robot lkp@intel.com
All warnings (new ones prefixed by >>):
In file included from fs/ext4/inode-test.c:7: fs/ext4/inode-test.c: In function 'ext4_inode_gen_params':
include/kunit/test.h:1735:58: warning: return discards 'const' qualifier from pointer target type [-Wdiscarded-qualifiers]
1735 | return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \
fs/ext4/inode-test.c:214:1: note: in expansion of macro 'KUNIT_ARRAY_PARAM' 214 | KUNIT_ARRAY_PARAM(ext4_inode, test_data); | ^~~~~~~~~~~~~~~~~
So this means we probably want to make the param_value, and the return and prev types of the generator "const void*".
Thanks, -- Marco
Okay, I'll fix this. Thanks!
Hi Arpitha,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on ext4/dev] [also build test ERROR on linus/master v5.10-rc1 next-20201030] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Arpitha-Raghunandan/kunit-Support-f... base: https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git dev config: x86_64-randconfig-a005-20201031 (attached as .config) compiler: clang version 12.0.0 (https://github.com/llvm/llvm-project 72ddd559b8aafef402091f8e192e025022e4ebef) reproduce (this is a W=1 build): wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross # install x86_64 cross compiling tool for clang build # apt-get install binutils-x86-64-linux-gnu # https://github.com/0day-ci/linux/commit/2de1e52708cd83d1dc4c718876683f680904... git remote add linux-review https://github.com/0day-ci/linux git fetch --no-tags linux-review Arpitha-Raghunandan/kunit-Support-for-Parameterized-Testing/20201028-015018 git checkout 2de1e52708cd83d1dc4c718876683f6809045a98 # save the attached .config to linux build tree COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=x86_64
If you fix the issue, kindly add following tag as appropriate Reported-by: kernel test robot lkp@intel.com
All errors (new ones prefixed by >>):
fs/ext4/inode-test.c:214:1: error: returning 'typeof ((test_data)[0]) *' (aka 'const struct timestamp_expectation *') from a function with result type 'void *' discards qualifiers [-Werror,-Wincompatible-pointer-types-discards-qualifiers]
KUNIT_ARRAY_PARAM(ext4_inode, test_data); ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ include/kunit/test.h:1735:10: note: expanded from macro 'KUNIT_ARRAY_PARAM' return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \ ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 1 error generated.
vim +214 fs/ext4/inode-test.c
213
214 KUNIT_ARRAY_PARAM(ext4_inode, test_data);
215
--- 0-DAY CI Kernel Test Service, Intel Corporation https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
On Tue, 27 Oct 2020 at 18:47, Arpitha Raghunandan 98.arpi@gmail.com wrote:
Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com
Changes v3->v4:
- Rename kunit variables
- Rename generator function helper macro
- Add documentation for generator approach
- Display test case name in case of failure along with param index
Changes v2->v3:
- Modifictaion of generator macro and method
Changes v1->v2:
- Use of a generator method to access test case parameters
include/kunit/test.h | 34 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 21 ++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ec2307ee9bb0 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,13 @@ struct kunit;
- @run_case: the function representing the actual test case.
- @name: the name of the test case.
- @generate_params: the generator function for parameterized tests.
- The generator function is used to lazily generate a series of
- arbitrarily typed values that fit into a void*. The argument @prev
- is the previously returned value, which should be used to derive the
- next value; @prev is set to NULL on the initial generator call.
- When no more values are available, the generator must return NULL.
Hmm, should this really be the first paragraph? I think it should be the paragraph before "Example:" maybe. But then that paragraph should refer to generate_params e.g. "The generator function @generate_params is used to ........".
The other option you have is to move this paragraph to the kernel-doc comment for KUNIT_CASE_PARAM, which seems to be missing a kernel-doc comment.
- A test case is a function with the signature,
- ``void (*)(struct kunit *)``
@@ -141,6 +148,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name;
void* (*generate_params)(void *prev); /* private: internal use only. */ bool success;
@@ -162,6 +170,9 @@ static inline char *kunit_status_to_string(bool status)
- &struct kunit_case for an example on how to use it.
*/ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
I.e. create a new kernel-doc comment for KUNIT_CASE_PARAM here, and simply move the paragraph describing the generator protocol into that comment.
+#define KUNIT_CASE_PARAM(test_name, gen_params) \
{ .run_case = test_name, .name = #test_name, \
.generate_params = gen_params }
/**
- struct kunit_suite - describes a related collection of &struct kunit_case
@@ -208,6 +219,15 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch;
/* param_value points to test case parameters in parameterized tests */
Hmm, not quite: param_value is the current parameter value for a test case. Most likely it's a pointer, but it doesn't need to be.
void *param_value;
/*
* param_index stores the index of the parameter in
* parameterized tests. param_index + 1 is printed
* to indicate the parameter that causes the test
* to fail in case of test failure.
*/
I think this comment needs to be reformatted, because you can use at the very least use 80 cols per line. (If you use vim, visual select and do 'gq'.)
int param_index; /* * success starts as true, and may only be set to false during a * test case; thus, it is safe to update this across multiple
@@ -1742,4 +1762,18 @@ do { \ fmt, \ ##__VA_ARGS__)
+/**
- KUNIT_ARRAY_PARAM() - Helper method for test parameter generators
required in parameterized tests.
- @name: prefix of the name for the test parameter generator function.
It will be suffixed by "_gen_params".
- @array: a user-supplied pointer to an array of test parameters.
- */
+#define KUNIT_ARRAY_PARAM(name, array) \
static void *name##_gen_params(void *prev) \
{ \
typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \
return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \
}
#endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..8ad908b61494 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -127,6 +127,12 @@ unsigned int kunit_test_case_num(struct kunit_suite *suite, } EXPORT_SYMBOL_GPL(kunit_test_case_num);
+static void kunit_print_failed_param(struct kunit *test) +{
kunit_err(test, "\n\tTest failed at:\n\ttest case: %s\n\tparameter: %d\n",
test->name, test->param_index + 1);
+}
Hmm, perhaps I wasn't clear, but I think I also misunderstood how the test case successes are presented: they are not, and it's all bunched into a single test case.
Firstly, kunit_err() already prints the test name, so if we want something like " # : the_test_case_name: failed at parameter #X", simply having
kunit_err(test, "failed at parameter #%d\n", test->param_index + 1)
would be what you want.
But I think I missed that parameters do not actually produce a set of test cases (sorry for noticing late). I think in their current form, the parameterized tests would not be useful for my tests, because each of my tests have test cases that have specific init and exit functions. For each parameter, these would also need to run.
Ideally, each parameter produces its own independent test case "test_case#param_index". That way, CI systems will also be able to logically separate different test case params, simply because each param produced its own distinct test case.
So, for example, we would get a series of test cases from something like KUNIT_CASE_PARAM(test_case, foo_gen_params), and in the output we'd see:
ok X - test_case#1 ok X - test_case#2 ok X - test_case#3 ok X - test_case#4 ....
Would that make more sense?
That way we'd ensure that test-case specific initialization and cleanup done in init and exit functions is properly taken care of, and you wouldn't need kunit_print_failed_param().
AFAIK, for what I propose you'd have to modify kunit_print_ok_not_ok() (show param_index if parameterized test) and probably kunit_run_case_catch_errors() (generate params and set test->param_value and param_index).
Was there a reason why each param cannot be a distinct test case? If not, I think this would be more useful.
static void kunit_print_string_stream(struct kunit *test, struct string_stream *stream) { @@ -168,6 +174,8 @@ static void kunit_fail(struct kunit *test, struct kunit_assert *assert) assert->format(assert, stream);
kunit_print_string_stream(test, stream);
if (test->param_value)
kunit_print_failed_param(test); WARN_ON(string_stream_destroy(stream));
} @@ -239,7 +247,18 @@ static void kunit_run_case_internal(struct kunit *test, } }
test_case->run_case(test);
if (!test_case->generate_params) {
test_case->run_case(test);
} else {
test->param_value = test_case->generate_params(NULL);
test->param_index = 0;
while (test->param_value) {
test_case->run_case(test);
test->param_value = test_case->generate_params(test->param_value);
test->param_index++;
}
}
Thanks, -- Marco
On 28/10/20 12:51 am, Marco Elver wrote:
On Tue, 27 Oct 2020 at 18:47, Arpitha Raghunandan 98.arpi@gmail.com wrote:
Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com
Changes v3->v4:
- Rename kunit variables
- Rename generator function helper macro
- Add documentation for generator approach
- Display test case name in case of failure along with param index
Changes v2->v3:
- Modifictaion of generator macro and method
Changes v1->v2:
- Use of a generator method to access test case parameters
include/kunit/test.h | 34 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 21 ++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ec2307ee9bb0 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,13 @@ struct kunit;
- @run_case: the function representing the actual test case.
- @name: the name of the test case.
- @generate_params: the generator function for parameterized tests.
- The generator function is used to lazily generate a series of
- arbitrarily typed values that fit into a void*. The argument @prev
- is the previously returned value, which should be used to derive the
- next value; @prev is set to NULL on the initial generator call.
- When no more values are available, the generator must return NULL.
Hmm, should this really be the first paragraph? I think it should be the paragraph before "Example:" maybe. But then that paragraph should refer to generate_params e.g. "The generator function @generate_params is used to ........".
The other option you have is to move this paragraph to the kernel-doc comment for KUNIT_CASE_PARAM, which seems to be missing a kernel-doc comment.
- A test case is a function with the signature,
- ``void (*)(struct kunit *)``
@@ -141,6 +148,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name;
void* (*generate_params)(void *prev); /* private: internal use only. */ bool success;
@@ -162,6 +170,9 @@ static inline char *kunit_status_to_string(bool status)
- &struct kunit_case for an example on how to use it.
*/ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
I.e. create a new kernel-doc comment for KUNIT_CASE_PARAM here, and simply move the paragraph describing the generator protocol into that comment.
I will make this change.
+#define KUNIT_CASE_PARAM(test_name, gen_params) \
{ .run_case = test_name, .name = #test_name, \
.generate_params = gen_params }
/**
- struct kunit_suite - describes a related collection of &struct kunit_case
@@ -208,6 +219,15 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch;
/* param_value points to test case parameters in parameterized tests */
Hmm, not quite: param_value is the current parameter value for a test case. Most likely it's a pointer, but it doesn't need to be.
void *param_value;
/*
* param_index stores the index of the parameter in
* parameterized tests. param_index + 1 is printed
* to indicate the parameter that causes the test
* to fail in case of test failure.
*/
I think this comment needs to be reformatted, because you can use at the very least use 80 cols per line. (If you use vim, visual select and do 'gq'.)
int param_index; /* * success starts as true, and may only be set to false during a * test case; thus, it is safe to update this across multiple
@@ -1742,4 +1762,18 @@ do { \ fmt, \ ##__VA_ARGS__)
+/**
- KUNIT_ARRAY_PARAM() - Helper method for test parameter generators
required in parameterized tests.
- @name: prefix of the name for the test parameter generator function.
It will be suffixed by "_gen_params".
- @array: a user-supplied pointer to an array of test parameters.
- */
+#define KUNIT_ARRAY_PARAM(name, array) \
static void *name##_gen_params(void *prev) \
{ \
typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \
return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \
}
#endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..8ad908b61494 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -127,6 +127,12 @@ unsigned int kunit_test_case_num(struct kunit_suite *suite, } EXPORT_SYMBOL_GPL(kunit_test_case_num);
+static void kunit_print_failed_param(struct kunit *test) +{
kunit_err(test, "\n\tTest failed at:\n\ttest case: %s\n\tparameter: %d\n",
test->name, test->param_index + 1);
+}
Hmm, perhaps I wasn't clear, but I think I also misunderstood how the test case successes are presented: they are not, and it's all bunched into a single test case.
Firstly, kunit_err() already prints the test name, so if we want something like " # : the_test_case_name: failed at parameter #X", simply having
kunit_err(test, "failed at parameter #%d\n", test->param_index + 1)
would be what you want.
But I think I missed that parameters do not actually produce a set of test cases (sorry for noticing late). I think in their current form, the parameterized tests would not be useful for my tests, because each of my tests have test cases that have specific init and exit functions. For each parameter, these would also need to run.
Ideally, each parameter produces its own independent test case "test_case#param_index". That way, CI systems will also be able to logically separate different test case params, simply because each param produced its own distinct test case.
So, for example, we would get a series of test cases from something like KUNIT_CASE_PARAM(test_case, foo_gen_params), and in the output we'd see:
ok X - test_case#1 ok X - test_case#2 ok X - test_case#3 ok X - test_case#4 ....
Would that make more sense?
That way we'd ensure that test-case specific initialization and cleanup done in init and exit functions is properly taken care of, and you wouldn't need kunit_print_failed_param().
AFAIK, for what I propose you'd have to modify kunit_print_ok_not_ok() (show param_index if parameterized test) and probably kunit_run_case_catch_errors() (generate params and set test->param_value and param_index).
Was there a reason why each param cannot be a distinct test case? If not, I think this would be more useful.
Oh, I hadn't considered this earlier. I will try it out for the next version.
static void kunit_print_string_stream(struct kunit *test, struct string_stream *stream) { @@ -168,6 +174,8 @@ static void kunit_fail(struct kunit *test, struct kunit_assert *assert) assert->format(assert, stream);
kunit_print_string_stream(test, stream);
if (test->param_value)
kunit_print_failed_param(test); WARN_ON(string_stream_destroy(stream));
} @@ -239,7 +247,18 @@ static void kunit_run_case_internal(struct kunit *test, } }
test_case->run_case(test);
if (!test_case->generate_params) {
test_case->run_case(test);
} else {
test->param_value = test_case->generate_params(NULL);
test->param_index = 0;
while (test->param_value) {
test_case->run_case(test);
test->param_value = test_case->generate_params(test->param_value);
test->param_index++;
}
}
Thanks, -- Marco
I'll make all the suggested changes. Thanks!
On 28/10/20 12:51 am, Marco Elver wrote:
On Tue, 27 Oct 2020 at 18:47, Arpitha Raghunandan 98.arpi@gmail.com wrote:
Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com
Changes v3->v4:
- Rename kunit variables
- Rename generator function helper macro
- Add documentation for generator approach
- Display test case name in case of failure along with param index
Changes v2->v3:
- Modifictaion of generator macro and method
Changes v1->v2:
- Use of a generator method to access test case parameters
include/kunit/test.h | 34 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 21 ++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ec2307ee9bb0 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,13 @@ struct kunit;
- @run_case: the function representing the actual test case.
- @name: the name of the test case.
- @generate_params: the generator function for parameterized tests.
- The generator function is used to lazily generate a series of
- arbitrarily typed values that fit into a void*. The argument @prev
- is the previously returned value, which should be used to derive the
- next value; @prev is set to NULL on the initial generator call.
- When no more values are available, the generator must return NULL.
Hmm, should this really be the first paragraph? I think it should be the paragraph before "Example:" maybe. But then that paragraph should refer to generate_params e.g. "The generator function @generate_params is used to ........".
The other option you have is to move this paragraph to the kernel-doc comment for KUNIT_CASE_PARAM, which seems to be missing a kernel-doc comment.
- A test case is a function with the signature,
- ``void (*)(struct kunit *)``
@@ -141,6 +148,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name;
void* (*generate_params)(void *prev); /* private: internal use only. */ bool success;
@@ -162,6 +170,9 @@ static inline char *kunit_status_to_string(bool status)
- &struct kunit_case for an example on how to use it.
*/ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
I.e. create a new kernel-doc comment for KUNIT_CASE_PARAM here, and simply move the paragraph describing the generator protocol into that comment.
+#define KUNIT_CASE_PARAM(test_name, gen_params) \
{ .run_case = test_name, .name = #test_name, \
.generate_params = gen_params }
/**
- struct kunit_suite - describes a related collection of &struct kunit_case
@@ -208,6 +219,15 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch;
/* param_value points to test case parameters in parameterized tests */
Hmm, not quite: param_value is the current parameter value for a test case. Most likely it's a pointer, but it doesn't need to be.
void *param_value;
/*
* param_index stores the index of the parameter in
* parameterized tests. param_index + 1 is printed
* to indicate the parameter that causes the test
* to fail in case of test failure.
*/
I think this comment needs to be reformatted, because you can use at the very least use 80 cols per line. (If you use vim, visual select and do 'gq'.)
int param_index; /* * success starts as true, and may only be set to false during a * test case; thus, it is safe to update this across multiple
@@ -1742,4 +1762,18 @@ do { \ fmt, \ ##__VA_ARGS__)
+/**
- KUNIT_ARRAY_PARAM() - Helper method for test parameter generators
required in parameterized tests.
- @name: prefix of the name for the test parameter generator function.
It will be suffixed by "_gen_params".
- @array: a user-supplied pointer to an array of test parameters.
- */
+#define KUNIT_ARRAY_PARAM(name, array) \
static void *name##_gen_params(void *prev) \
{ \
typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \
return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \
}
#endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..8ad908b61494 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -127,6 +127,12 @@ unsigned int kunit_test_case_num(struct kunit_suite *suite, } EXPORT_SYMBOL_GPL(kunit_test_case_num);
+static void kunit_print_failed_param(struct kunit *test) +{
kunit_err(test, "\n\tTest failed at:\n\ttest case: %s\n\tparameter: %d\n",
test->name, test->param_index + 1);
+}
Hmm, perhaps I wasn't clear, but I think I also misunderstood how the test case successes are presented: they are not, and it's all bunched into a single test case.
Firstly, kunit_err() already prints the test name, so if we want something like " # : the_test_case_name: failed at parameter #X", simply having
kunit_err(test, "failed at parameter #%d\n", test->param_index + 1)
would be what you want.
But I think I missed that parameters do not actually produce a set of test cases (sorry for noticing late). I think in their current form, the parameterized tests would not be useful for my tests, because each of my tests have test cases that have specific init and exit functions. For each parameter, these would also need to run.
Ideally, each parameter produces its own independent test case "test_case#param_index". That way, CI systems will also be able to logically separate different test case params, simply because each param produced its own distinct test case.
So, for example, we would get a series of test cases from something like KUNIT_CASE_PARAM(test_case, foo_gen_params), and in the output we'd see:
ok X - test_case#1 ok X - test_case#2 ok X - test_case#3 ok X - test_case#4 ....
Would that make more sense?
That way we'd ensure that test-case specific initialization and cleanup done in init and exit functions is properly taken care of, and you wouldn't need kunit_print_failed_param().
AFAIK, for what I propose you'd have to modify kunit_print_ok_not_ok() (show param_index if parameterized test) and probably kunit_run_case_catch_errors() (generate params and set test->param_value and param_index).
Was there a reason why each param cannot be a distinct test case? If not, I think this would be more useful.
I tried adding support to run each parameter as a distinct test case by making changes to kunit_run_case_catch_errors(). The issue here is that since the results are displayed in KTAP format, this change will result in each parameter being considered a subtest of another subtest (test case in KUnit). To make this work, a lot of changes in other parts will be required, and it will get complicated. Running all parameters as one test case seems to be a better option right now. So for now, I will modify what is displayed by kunit_err() in case of test failure.
static void kunit_print_string_stream(struct kunit *test, struct string_stream *stream) { @@ -168,6 +174,8 @@ static void kunit_fail(struct kunit *test, struct kunit_assert *assert) assert->format(assert, stream);
kunit_print_string_stream(test, stream);
if (test->param_value)
kunit_print_failed_param(test); WARN_ON(string_stream_destroy(stream));
} @@ -239,7 +247,18 @@ static void kunit_run_case_internal(struct kunit *test, } }
test_case->run_case(test);
if (!test_case->generate_params) {
test_case->run_case(test);
} else {
test->param_value = test_case->generate_params(NULL);
test->param_index = 0;
while (test->param_value) {
test_case->run_case(test);
test->param_value = test_case->generate_params(test->param_value);
test->param_index++;
}
}
Thanks, -- Marco
On Thu, 5 Nov 2020 at 08:32, Arpitha Raghunandan 98.arpi@gmail.com wrote:
On 28/10/20 12:51 am, Marco Elver wrote:
On Tue, 27 Oct 2020 at 18:47, Arpitha Raghunandan 98.arpi@gmail.com wrote:
Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com
Changes v3->v4:
- Rename kunit variables
- Rename generator function helper macro
- Add documentation for generator approach
- Display test case name in case of failure along with param index
Changes v2->v3:
- Modifictaion of generator macro and method
Changes v1->v2:
- Use of a generator method to access test case parameters
include/kunit/test.h | 34 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 21 ++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ec2307ee9bb0 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,13 @@ struct kunit;
- @run_case: the function representing the actual test case.
- @name: the name of the test case.
- @generate_params: the generator function for parameterized tests.
- The generator function is used to lazily generate a series of
- arbitrarily typed values that fit into a void*. The argument @prev
- is the previously returned value, which should be used to derive the
- next value; @prev is set to NULL on the initial generator call.
- When no more values are available, the generator must return NULL.
Hmm, should this really be the first paragraph? I think it should be the paragraph before "Example:" maybe. But then that paragraph should refer to generate_params e.g. "The generator function @generate_params is used to ........".
The other option you have is to move this paragraph to the kernel-doc comment for KUNIT_CASE_PARAM, which seems to be missing a kernel-doc comment.
- A test case is a function with the signature,
- ``void (*)(struct kunit *)``
@@ -141,6 +148,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name;
void* (*generate_params)(void *prev); /* private: internal use only. */ bool success;
@@ -162,6 +170,9 @@ static inline char *kunit_status_to_string(bool status)
- &struct kunit_case for an example on how to use it.
*/ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
I.e. create a new kernel-doc comment for KUNIT_CASE_PARAM here, and simply move the paragraph describing the generator protocol into that comment.
+#define KUNIT_CASE_PARAM(test_name, gen_params) \
{ .run_case = test_name, .name = #test_name, \
.generate_params = gen_params }
/**
- struct kunit_suite - describes a related collection of &struct kunit_case
@@ -208,6 +219,15 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch;
/* param_value points to test case parameters in parameterized tests */
Hmm, not quite: param_value is the current parameter value for a test case. Most likely it's a pointer, but it doesn't need to be.
void *param_value;
/*
* param_index stores the index of the parameter in
* parameterized tests. param_index + 1 is printed
* to indicate the parameter that causes the test
* to fail in case of test failure.
*/
I think this comment needs to be reformatted, because you can use at the very least use 80 cols per line. (If you use vim, visual select and do 'gq'.)
int param_index; /* * success starts as true, and may only be set to false during a * test case; thus, it is safe to update this across multiple
@@ -1742,4 +1762,18 @@ do { \ fmt, \ ##__VA_ARGS__)
+/**
- KUNIT_ARRAY_PARAM() - Helper method for test parameter generators
required in parameterized tests.
- @name: prefix of the name for the test parameter generator function.
It will be suffixed by "_gen_params".
- @array: a user-supplied pointer to an array of test parameters.
- */
+#define KUNIT_ARRAY_PARAM(name, array) \
static void *name##_gen_params(void *prev) \
{ \
typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \
return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \
}
#endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..8ad908b61494 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -127,6 +127,12 @@ unsigned int kunit_test_case_num(struct kunit_suite *suite, } EXPORT_SYMBOL_GPL(kunit_test_case_num);
+static void kunit_print_failed_param(struct kunit *test) +{
kunit_err(test, "\n\tTest failed at:\n\ttest case: %s\n\tparameter: %d\n",
test->name, test->param_index + 1);
+}
Hmm, perhaps I wasn't clear, but I think I also misunderstood how the test case successes are presented: they are not, and it's all bunched into a single test case.
Firstly, kunit_err() already prints the test name, so if we want something like " # : the_test_case_name: failed at parameter #X", simply having
kunit_err(test, "failed at parameter #%d\n", test->param_index + 1)
would be what you want.
But I think I missed that parameters do not actually produce a set of test cases (sorry for noticing late). I think in their current form, the parameterized tests would not be useful for my tests, because each of my tests have test cases that have specific init and exit functions. For each parameter, these would also need to run.
Ideally, each parameter produces its own independent test case "test_case#param_index". That way, CI systems will also be able to logically separate different test case params, simply because each param produced its own distinct test case.
So, for example, we would get a series of test cases from something like KUNIT_CASE_PARAM(test_case, foo_gen_params), and in the output we'd see:
ok X - test_case#1 ok X - test_case#2 ok X - test_case#3 ok X - test_case#4 ....
Would that make more sense?
That way we'd ensure that test-case specific initialization and cleanup done in init and exit functions is properly taken care of, and you wouldn't need kunit_print_failed_param().
AFAIK, for what I propose you'd have to modify kunit_print_ok_not_ok() (show param_index if parameterized test) and probably kunit_run_case_catch_errors() (generate params and set test->param_value and param_index).
Was there a reason why each param cannot be a distinct test case? If not, I think this would be more useful.
I tried adding support to run each parameter as a distinct test case by making changes to kunit_run_case_catch_errors(). The issue here is that since the results are displayed in KTAP format, this change will result in each parameter being considered a subtest of another subtest (test case in KUnit).
Do you have example output? That might help understand what's going on.
To make this work, a lot of changes in other parts will be required, and it will get complicated. Running all parameters as one test case seems to be a better option right now. So for now, I will modify what is displayed by kunit_err() in case of test failure.
static void kunit_print_string_stream(struct kunit *test, struct string_stream *stream) { @@ -168,6 +174,8 @@ static void kunit_fail(struct kunit *test, struct kunit_assert *assert) assert->format(assert, stream);
kunit_print_string_stream(test, stream);
if (test->param_value)
kunit_print_failed_param(test); WARN_ON(string_stream_destroy(stream));
} @@ -239,7 +247,18 @@ static void kunit_run_case_internal(struct kunit *test, } }
test_case->run_case(test);
if (!test_case->generate_params) {
test_case->run_case(test);
} else {
test->param_value = test_case->generate_params(NULL);
test->param_index = 0;
while (test->param_value) {
test_case->run_case(test);
test->param_value = test_case->generate_params(test->param_value);
test->param_index++;
}
}
Thanks, -- Marco
-- You received this message because you are subscribed to the Google Groups "KUnit Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to kunit-dev+unsubscribe@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/kunit-dev/73c4e46c-10f1-9362-b4fb-94ea9d74....
On 05/11/20 2:00 pm, Marco Elver wrote:
On Thu, 5 Nov 2020 at 08:32, Arpitha Raghunandan 98.arpi@gmail.com wrote:
On 28/10/20 12:51 am, Marco Elver wrote:
On Tue, 27 Oct 2020 at 18:47, Arpitha Raghunandan 98.arpi@gmail.com wrote:
Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com
Changes v3->v4:
- Rename kunit variables
- Rename generator function helper macro
- Add documentation for generator approach
- Display test case name in case of failure along with param index
Changes v2->v3:
- Modifictaion of generator macro and method
Changes v1->v2:
- Use of a generator method to access test case parameters
include/kunit/test.h | 34 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 21 ++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ec2307ee9bb0 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,13 @@ struct kunit;
- @run_case: the function representing the actual test case.
- @name: the name of the test case.
- @generate_params: the generator function for parameterized tests.
- The generator function is used to lazily generate a series of
- arbitrarily typed values that fit into a void*. The argument @prev
- is the previously returned value, which should be used to derive the
- next value; @prev is set to NULL on the initial generator call.
- When no more values are available, the generator must return NULL.
Hmm, should this really be the first paragraph? I think it should be the paragraph before "Example:" maybe. But then that paragraph should refer to generate_params e.g. "The generator function @generate_params is used to ........".
The other option you have is to move this paragraph to the kernel-doc comment for KUNIT_CASE_PARAM, which seems to be missing a kernel-doc comment.
- A test case is a function with the signature,
- ``void (*)(struct kunit *)``
@@ -141,6 +148,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name;
void* (*generate_params)(void *prev); /* private: internal use only. */ bool success;
@@ -162,6 +170,9 @@ static inline char *kunit_status_to_string(bool status)
- &struct kunit_case for an example on how to use it.
*/ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
I.e. create a new kernel-doc comment for KUNIT_CASE_PARAM here, and simply move the paragraph describing the generator protocol into that comment.
+#define KUNIT_CASE_PARAM(test_name, gen_params) \
{ .run_case = test_name, .name = #test_name, \
.generate_params = gen_params }
/**
- struct kunit_suite - describes a related collection of &struct kunit_case
@@ -208,6 +219,15 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch;
/* param_value points to test case parameters in parameterized tests */
Hmm, not quite: param_value is the current parameter value for a test case. Most likely it's a pointer, but it doesn't need to be.
void *param_value;
/*
* param_index stores the index of the parameter in
* parameterized tests. param_index + 1 is printed
* to indicate the parameter that causes the test
* to fail in case of test failure.
*/
I think this comment needs to be reformatted, because you can use at the very least use 80 cols per line. (If you use vim, visual select and do 'gq'.)
int param_index; /* * success starts as true, and may only be set to false during a * test case; thus, it is safe to update this across multiple
@@ -1742,4 +1762,18 @@ do { \ fmt, \ ##__VA_ARGS__)
+/**
- KUNIT_ARRAY_PARAM() - Helper method for test parameter generators
required in parameterized tests.
- @name: prefix of the name for the test parameter generator function.
It will be suffixed by "_gen_params".
- @array: a user-supplied pointer to an array of test parameters.
- */
+#define KUNIT_ARRAY_PARAM(name, array) \
static void *name##_gen_params(void *prev) \
{ \
typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \
return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \
}
#endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..8ad908b61494 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -127,6 +127,12 @@ unsigned int kunit_test_case_num(struct kunit_suite *suite, } EXPORT_SYMBOL_GPL(kunit_test_case_num);
+static void kunit_print_failed_param(struct kunit *test) +{
kunit_err(test, "\n\tTest failed at:\n\ttest case: %s\n\tparameter: %d\n",
test->name, test->param_index + 1);
+}
Hmm, perhaps I wasn't clear, but I think I also misunderstood how the test case successes are presented: they are not, and it's all bunched into a single test case.
Firstly, kunit_err() already prints the test name, so if we want something like " # : the_test_case_name: failed at parameter #X", simply having
kunit_err(test, "failed at parameter #%d\n", test->param_index + 1)
would be what you want.
But I think I missed that parameters do not actually produce a set of test cases (sorry for noticing late). I think in their current form, the parameterized tests would not be useful for my tests, because each of my tests have test cases that have specific init and exit functions. For each parameter, these would also need to run.
Ideally, each parameter produces its own independent test case "test_case#param_index". That way, CI systems will also be able to logically separate different test case params, simply because each param produced its own distinct test case.
So, for example, we would get a series of test cases from something like KUNIT_CASE_PARAM(test_case, foo_gen_params), and in the output we'd see:
ok X - test_case#1 ok X - test_case#2 ok X - test_case#3 ok X - test_case#4 ....
Would that make more sense?
That way we'd ensure that test-case specific initialization and cleanup done in init and exit functions is properly taken care of, and you wouldn't need kunit_print_failed_param().
AFAIK, for what I propose you'd have to modify kunit_print_ok_not_ok() (show param_index if parameterized test) and probably kunit_run_case_catch_errors() (generate params and set test->param_value and param_index).
Was there a reason why each param cannot be a distinct test case? If not, I think this would be more useful.
I tried adding support to run each parameter as a distinct test case by making changes to kunit_run_case_catch_errors(). The issue here is that since the results are displayed in KTAP format, this change will result in each parameter being considered a subtest of another subtest (test case in KUnit).
Do you have example output? That might help understand what's going on.
The change that I tried can be seen here (based on the v4 patch): https://gist.github.com/arpi-r/4822899087ca4cc34572ed9e45cc5fee.
Using the kunit tool, I get this error:
[19:20:41] [ERROR] expected 7 test suites, but got -1 [ERROR] no tests run! [19:20:41] ============================================================ [19:20:41] Testing complete. 0 tests run. 0 failed. 0 crashed.
But this error is only because of how the tool displays the results. The test actually does run, as can be seen in the dmesg output:
TAP version 14 1..7 # Subtest: ext4_inode_test 1..1 ok 1 - inode_test_xtimestamp_decoding 1 ok 1 - inode_test_xtimestamp_decoding 2 ok 1 - inode_test_xtimestamp_decoding 3 ok 1 - inode_test_xtimestamp_decoding 4 ok 1 - inode_test_xtimestamp_decoding 5 ok 1 - inode_test_xtimestamp_decoding 6 ok 1 - inode_test_xtimestamp_decoding 7 ok 1 - inode_test_xtimestamp_decoding 8 ok 1 - inode_test_xtimestamp_decoding 9 ok 1 - inode_test_xtimestamp_decoding 10 ok 1 - inode_test_xtimestamp_decoding 11 ok 1 - inode_test_xtimestamp_decoding 12 ok 1 - inode_test_xtimestamp_decoding 13 ok 1 - inode_test_xtimestamp_decoding 14 ok 1 - inode_test_xtimestamp_decoding 15 ok 1 - inode_test_xtimestamp_decoding 16 ok 1 - ext4_inode_test (followed by other kunit test outputs)
To make this work, a lot of changes in other parts will be required, and it will get complicated. Running all parameters as one test case seems to be a better option right now. So for now, I will modify what is displayed by kunit_err() in case of test failure.
static void kunit_print_string_stream(struct kunit *test, struct string_stream *stream) { @@ -168,6 +174,8 @@ static void kunit_fail(struct kunit *test, struct kunit_assert *assert) assert->format(assert, stream);
kunit_print_string_stream(test, stream);
if (test->param_value)
kunit_print_failed_param(test); WARN_ON(string_stream_destroy(stream));
} @@ -239,7 +247,18 @@ static void kunit_run_case_internal(struct kunit *test, } }
test_case->run_case(test);
if (!test_case->generate_params) {
test_case->run_case(test);
} else {
test->param_value = test_case->generate_params(NULL);
test->param_index = 0;
while (test->param_value) {
test_case->run_case(test);
test->param_value = test_case->generate_params(test->param_value);
test->param_index++;
}
}
Thanks, -- Marco
-- You received this message because you are subscribed to the Google Groups "KUnit Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to kunit-dev+unsubscribe@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/kunit-dev/73c4e46c-10f1-9362-b4fb-94ea9d74....
On Thu, 5 Nov 2020 at 15:30, Arpitha Raghunandan 98.arpi@gmail.com wrote:
On 05/11/20 2:00 pm, Marco Elver wrote:
On Thu, 5 Nov 2020 at 08:32, Arpitha Raghunandan 98.arpi@gmail.com wrote:
On 28/10/20 12:51 am, Marco Elver wrote:
On Tue, 27 Oct 2020 at 18:47, Arpitha Raghunandan 98.arpi@gmail.com wrote:
Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com
Changes v3->v4:
- Rename kunit variables
- Rename generator function helper macro
- Add documentation for generator approach
- Display test case name in case of failure along with param index
Changes v2->v3:
- Modifictaion of generator macro and method
Changes v1->v2:
- Use of a generator method to access test case parameters
include/kunit/test.h | 34 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 21 ++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ec2307ee9bb0 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,13 @@ struct kunit;
- @run_case: the function representing the actual test case.
- @name: the name of the test case.
- @generate_params: the generator function for parameterized tests.
- The generator function is used to lazily generate a series of
- arbitrarily typed values that fit into a void*. The argument @prev
- is the previously returned value, which should be used to derive the
- next value; @prev is set to NULL on the initial generator call.
- When no more values are available, the generator must return NULL.
Hmm, should this really be the first paragraph? I think it should be the paragraph before "Example:" maybe. But then that paragraph should refer to generate_params e.g. "The generator function @generate_params is used to ........".
The other option you have is to move this paragraph to the kernel-doc comment for KUNIT_CASE_PARAM, which seems to be missing a kernel-doc comment.
- A test case is a function with the signature,
- ``void (*)(struct kunit *)``
@@ -141,6 +148,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name;
void* (*generate_params)(void *prev); /* private: internal use only. */ bool success;
@@ -162,6 +170,9 @@ static inline char *kunit_status_to_string(bool status)
- &struct kunit_case for an example on how to use it.
*/ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
I.e. create a new kernel-doc comment for KUNIT_CASE_PARAM here, and simply move the paragraph describing the generator protocol into that comment.
+#define KUNIT_CASE_PARAM(test_name, gen_params) \
{ .run_case = test_name, .name = #test_name, \
.generate_params = gen_params }
/**
- struct kunit_suite - describes a related collection of &struct kunit_case
@@ -208,6 +219,15 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch;
/* param_value points to test case parameters in parameterized tests */
Hmm, not quite: param_value is the current parameter value for a test case. Most likely it's a pointer, but it doesn't need to be.
void *param_value;
/*
* param_index stores the index of the parameter in
* parameterized tests. param_index + 1 is printed
* to indicate the parameter that causes the test
* to fail in case of test failure.
*/
I think this comment needs to be reformatted, because you can use at the very least use 80 cols per line. (If you use vim, visual select and do 'gq'.)
int param_index; /* * success starts as true, and may only be set to false during a * test case; thus, it is safe to update this across multiple
@@ -1742,4 +1762,18 @@ do { \ fmt, \ ##__VA_ARGS__)
+/**
- KUNIT_ARRAY_PARAM() - Helper method for test parameter generators
required in parameterized tests.
- @name: prefix of the name for the test parameter generator function.
It will be suffixed by "_gen_params".
- @array: a user-supplied pointer to an array of test parameters.
- */
+#define KUNIT_ARRAY_PARAM(name, array) \
static void *name##_gen_params(void *prev) \
{ \
typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \
return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \
}
#endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..8ad908b61494 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -127,6 +127,12 @@ unsigned int kunit_test_case_num(struct kunit_suite *suite, } EXPORT_SYMBOL_GPL(kunit_test_case_num);
+static void kunit_print_failed_param(struct kunit *test) +{
kunit_err(test, "\n\tTest failed at:\n\ttest case: %s\n\tparameter: %d\n",
test->name, test->param_index + 1);
+}
Hmm, perhaps I wasn't clear, but I think I also misunderstood how the test case successes are presented: they are not, and it's all bunched into a single test case.
Firstly, kunit_err() already prints the test name, so if we want something like " # : the_test_case_name: failed at parameter #X", simply having
kunit_err(test, "failed at parameter #%d\n", test->param_index + 1)
would be what you want.
But I think I missed that parameters do not actually produce a set of test cases (sorry for noticing late). I think in their current form, the parameterized tests would not be useful for my tests, because each of my tests have test cases that have specific init and exit functions. For each parameter, these would also need to run.
Ideally, each parameter produces its own independent test case "test_case#param_index". That way, CI systems will also be able to logically separate different test case params, simply because each param produced its own distinct test case.
So, for example, we would get a series of test cases from something like KUNIT_CASE_PARAM(test_case, foo_gen_params), and in the output we'd see:
ok X - test_case#1 ok X - test_case#2 ok X - test_case#3 ok X - test_case#4 ....
Would that make more sense?
That way we'd ensure that test-case specific initialization and cleanup done in init and exit functions is properly taken care of, and you wouldn't need kunit_print_failed_param().
AFAIK, for what I propose you'd have to modify kunit_print_ok_not_ok() (show param_index if parameterized test) and probably kunit_run_case_catch_errors() (generate params and set test->param_value and param_index).
Was there a reason why each param cannot be a distinct test case? If not, I think this would be more useful.
I tried adding support to run each parameter as a distinct test case by making changes to kunit_run_case_catch_errors(). The issue here is that since the results are displayed in KTAP format, this change will result in each parameter being considered a subtest of another subtest (test case in KUnit).
Do you have example output? That might help understand what's going on.
The change that I tried can be seen here (based on the v4 patch): https://gist.github.com/arpi-r/4822899087ca4cc34572ed9e45cc5fee.
Using the kunit tool, I get this error:
[19:20:41] [ERROR] expected 7 test suites, but got -1 [ERROR] no tests run! [19:20:41] ============================================================ [19:20:41] Testing complete. 0 tests run. 0 failed. 0 crashed.
But this error is only because of how the tool displays the results. The test actually does run, as can be seen in the dmesg output:
TAP version 14 1..7 # Subtest: ext4_inode_test 1..1 ok 1 - inode_test_xtimestamp_decoding 1 ok 1 - inode_test_xtimestamp_decoding 2 ok 1 - inode_test_xtimestamp_decoding 3 ok 1 - inode_test_xtimestamp_decoding 4 ok 1 - inode_test_xtimestamp_decoding 5 ok 1 - inode_test_xtimestamp_decoding 6 ok 1 - inode_test_xtimestamp_decoding 7 ok 1 - inode_test_xtimestamp_decoding 8 ok 1 - inode_test_xtimestamp_decoding 9 ok 1 - inode_test_xtimestamp_decoding 10 ok 1 - inode_test_xtimestamp_decoding 11 ok 1 - inode_test_xtimestamp_decoding 12 ok 1 - inode_test_xtimestamp_decoding 13 ok 1 - inode_test_xtimestamp_decoding 14 ok 1 - inode_test_xtimestamp_decoding 15 ok 1 - inode_test_xtimestamp_decoding 16 ok 1 - ext4_inode_test (followed by other kunit test outputs)
Hmm, interesting. Let me play with your patch a bit.
One option is to just have the test case number increment as well, i.e. have this: | ok 1 - inode_test_xtimestamp_decoding#1 | ok 2 - inode_test_xtimestamp_decoding#2 | ok 3 - inode_test_xtimestamp_decoding#3 | ok 4 - inode_test_xtimestamp_decoding#4 | ok 5 - inode_test_xtimestamp_decoding#5 ...
Or is there something else I missed?
Thanks, -- Marco
On Thu, Nov 05, 2020 at 04:02PM +0100, Marco Elver wrote:
On Thu, 5 Nov 2020 at 15:30, Arpitha Raghunandan 98.arpi@gmail.com wrote:
[...]
I tried adding support to run each parameter as a distinct test case by making changes to kunit_run_case_catch_errors(). The issue here is that since the results are displayed in KTAP format, this change will result in each parameter being considered a subtest of another subtest (test case in KUnit).
Do you have example output? That might help understand what's going on.
The change that I tried can be seen here (based on the v4 patch): https://gist.github.com/arpi-r/4822899087ca4cc34572ed9e45cc5fee.
Using the kunit tool, I get this error:
[19:20:41] [ERROR] expected 7 test suites, but got -1 [ERROR] no tests run! [19:20:41] ============================================================ [19:20:41] Testing complete. 0 tests run. 0 failed. 0 crashed.
But this error is only because of how the tool displays the results. The test actually does run, as can be seen in the dmesg output:
TAP version 14 1..7 # Subtest: ext4_inode_test 1..1 ok 1 - inode_test_xtimestamp_decoding 1 ok 1 - inode_test_xtimestamp_decoding 2 ok 1 - inode_test_xtimestamp_decoding 3 ok 1 - inode_test_xtimestamp_decoding 4 ok 1 - inode_test_xtimestamp_decoding 5 ok 1 - inode_test_xtimestamp_decoding 6 ok 1 - inode_test_xtimestamp_decoding 7 ok 1 - inode_test_xtimestamp_decoding 8 ok 1 - inode_test_xtimestamp_decoding 9 ok 1 - inode_test_xtimestamp_decoding 10 ok 1 - inode_test_xtimestamp_decoding 11 ok 1 - inode_test_xtimestamp_decoding 12 ok 1 - inode_test_xtimestamp_decoding 13 ok 1 - inode_test_xtimestamp_decoding 14 ok 1 - inode_test_xtimestamp_decoding 15 ok 1 - inode_test_xtimestamp_decoding 16 ok 1 - ext4_inode_test (followed by other kunit test outputs)
Hmm, interesting. Let me play with your patch a bit.
One option is to just have the test case number increment as well, i.e. have this: | ok 1 - inode_test_xtimestamp_decoding#1 | ok 2 - inode_test_xtimestamp_decoding#2 | ok 3 - inode_test_xtimestamp_decoding#3 | ok 4 - inode_test_xtimestamp_decoding#4 | ok 5 - inode_test_xtimestamp_decoding#5 ...
Or is there something else I missed?
Right, so TAP wants the exact number of tests it will run ahead of time. In which case we can still put the results of each parameterized test in a diagnostic. Please see my proposed patch below, which still does proper initialization/destruction of each parameter case as if it was its own test case.
With it the output looks as follows:
| TAP version 14 | 1..6 | # Subtest: ext4_inode_test | 1..1 | # ok param#0 - inode_test_xtimestamp_decoding | # ok param#1 - inode_test_xtimestamp_decoding | # ok param#2 - inode_test_xtimestamp_decoding | # ok param#3 - inode_test_xtimestamp_decoding | # ok param#4 - inode_test_xtimestamp_decoding | # ok param#5 - inode_test_xtimestamp_decoding | # ok param#6 - inode_test_xtimestamp_decoding | # ok param#7 - inode_test_xtimestamp_decoding | # ok param#8 - inode_test_xtimestamp_decoding | # ok param#9 - inode_test_xtimestamp_decoding | # ok param#10 - inode_test_xtimestamp_decoding | # ok param#11 - inode_test_xtimestamp_decoding | # ok param#12 - inode_test_xtimestamp_decoding | # ok param#13 - inode_test_xtimestamp_decoding | # ok param#14 - inode_test_xtimestamp_decoding | # ok param#15 - inode_test_xtimestamp_decoding | ok 1 - inode_test_xtimestamp_decoding | ok 1 - ext4_inode_test
Would that be reasonable? If so, feel free to take the patch and test/adjust as required.
I'm not sure on the best format -- is there is a recommended format for parameterized test result output? If not, I suppose we can put anything we like into the diagnostic.
Thanks, -- Marco
------ >8 ------
From ccbf4e2e190a2d7c6a94a51c9b1fb3b9a940e578 Mon Sep 17 00:00:00 2001 From: Arpitha Raghunandan 98.arpi@gmail.com Date: Tue, 27 Oct 2020 23:16:30 +0530 Subject: [PATCH] kunit: Support for Parameterized Testing
Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com --- Changes v4->v5: - Update kernel-doc comments. - Use const void* for generator return and prev value types. - Add kernel-doc comment for KUNIT_ARRAY_PARAM. - Rework parameterized test case execution strategy: each parameter is executed as if it was its own test case, with its own test initialization and cleanup (init and exit are called, etc.). However, we cannot add new test cases per TAP protocol once we have already started execution. Instead, log the result of each parameter run as a diagnostic comment. Changes v3->v4: - Rename kunit variables - Rename generator function helper macro - Add documentation for generator approach - Display test case name in case of failure along with param index Changes v2->v3: - Modifictaion of generator macro and method Changes v1->v2: - Use of a generator method to access test case parameters --- include/kunit/test.h | 36 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 46 +++++++++++++++++++++++++++++++------------- 2 files changed, 69 insertions(+), 13 deletions(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ae5488a37e48 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,7 @@ struct kunit; * * @run_case: the function representing the actual test case. * @name: the name of the test case. + * @generate_params: the generator function for parameterized tests. * * A test case is a function with the signature, * ``void (*)(struct kunit *)`` @@ -141,6 +142,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name; + const void* (*generate_params)(const void *prev);
/* private: internal use only. */ bool success; @@ -163,6 +165,22 @@ static inline char *kunit_status_to_string(bool status) */ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
+/** + * KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case + * + * @test_name: a reference to a test case function. + * @gen_params: a reference to a parameter generator function. + * + * The generator function ``const void* gen_params(const void *prev)`` is used + * to lazily generate a series of arbitrarily typed values that fit into a + * void*. The argument @prev is the previously returned value, which should be + * used to derive the next value; @prev is set to NULL on the initial generator + * call. When no more values are available, the generator must return NULL. + */ +#define KUNIT_CASE_PARAM(test_name, gen_params) \ + { .run_case = test_name, .name = #test_name, \ + .generate_params = gen_params } + /** * struct kunit_suite - describes a related collection of &struct kunit_case * @@ -208,6 +226,10 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch; + /* param_value is the current parameter value for a test case. */ + const void *param_value; + /* param_index stores the index of the parameter in parameterized tests. */ + int param_index; /* * success starts as true, and may only be set to false during a * test case; thus, it is safe to update this across multiple @@ -1742,4 +1764,18 @@ do { \ fmt, \ ##__VA_ARGS__)
+/** + * KUNIT_ARRAY_PARAM() - Define test parameter generator from an array. + * @name: prefix for the test parameter generator function. + * @array: array of test parameters. + * + * Define function @name_gen_params which uses @array to generate parameters. + */ +#define KUNIT_ARRAY_PARAM(name, array) \ + static const void *name##_gen_params(const void *prev) \ + { \ + typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \ + return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \ + } + #endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..453ebe4da77d 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -325,29 +325,25 @@ static void kunit_catch_run_case(void *data) * occur in a test case and reports them as failures. */ static void kunit_run_case_catch_errors(struct kunit_suite *suite, - struct kunit_case *test_case) + struct kunit_case *test_case, + struct kunit *test) { struct kunit_try_catch_context context; struct kunit_try_catch *try_catch; - struct kunit test;
- kunit_init_test(&test, test_case->name, test_case->log); - try_catch = &test.try_catch; + kunit_init_test(test, test_case->name, test_case->log); + try_catch = &test->try_catch;
kunit_try_catch_init(try_catch, - &test, + test, kunit_try_run_case, kunit_catch_run_case); - context.test = &test; + context.test = test; context.suite = suite; context.test_case = test_case; kunit_try_catch_run(try_catch, &context);
- test_case->success = test.success; - - kunit_print_ok_not_ok(&test, true, test_case->success, - kunit_test_case_num(suite, test_case), - test_case->name); + test_case->success = test->success; }
int kunit_run_tests(struct kunit_suite *suite) @@ -356,8 +352,32 @@ int kunit_run_tests(struct kunit_suite *suite)
kunit_print_subtest_start(suite);
- kunit_suite_for_each_test_case(suite, test_case) - kunit_run_case_catch_errors(suite, test_case); + kunit_suite_for_each_test_case(suite, test_case) { + struct kunit test = { .param_value = NULL, .param_index = 0 }; + bool test_success = true; + + if (test_case->generate_params) + test.param_value = test_case->generate_params(NULL); + + do { + kunit_run_case_catch_errors(suite, test_case, &test); + test_success &= test_case->success; + + if (test_case->generate_params) { + kunit_log(KERN_INFO, &test, + KUNIT_SUBTEST_INDENT + "# %s param#%d - %s", + kunit_status_to_string(test.success), + test.param_index, test_case->name); + test.param_value = test_case->generate_params(test.param_value); + test.param_index++; + } + } while (test.param_value); + + kunit_print_ok_not_ok(&test, true, test_success, + kunit_test_case_num(suite, test_case), + test_case->name); + }
kunit_print_subtest_end(suite);
On 06/11/20 1:25 am, Marco Elver wrote:
On Thu, Nov 05, 2020 at 04:02PM +0100, Marco Elver wrote:
On Thu, 5 Nov 2020 at 15:30, Arpitha Raghunandan 98.arpi@gmail.com wrote:
[...]
I tried adding support to run each parameter as a distinct test case by making changes to kunit_run_case_catch_errors(). The issue here is that since the results are displayed in KTAP format, this change will result in each parameter being considered a subtest of another subtest (test case in KUnit).
Do you have example output? That might help understand what's going on.
The change that I tried can be seen here (based on the v4 patch): https://gist.github.com/arpi-r/4822899087ca4cc34572ed9e45cc5fee.
Using the kunit tool, I get this error:
[19:20:41] [ERROR] expected 7 test suites, but got -1 [ERROR] no tests run! [19:20:41] ============================================================ [19:20:41] Testing complete. 0 tests run. 0 failed. 0 crashed.
But this error is only because of how the tool displays the results. The test actually does run, as can be seen in the dmesg output:
TAP version 14 1..7 # Subtest: ext4_inode_test 1..1 ok 1 - inode_test_xtimestamp_decoding 1 ok 1 - inode_test_xtimestamp_decoding 2 ok 1 - inode_test_xtimestamp_decoding 3 ok 1 - inode_test_xtimestamp_decoding 4 ok 1 - inode_test_xtimestamp_decoding 5 ok 1 - inode_test_xtimestamp_decoding 6 ok 1 - inode_test_xtimestamp_decoding 7 ok 1 - inode_test_xtimestamp_decoding 8 ok 1 - inode_test_xtimestamp_decoding 9 ok 1 - inode_test_xtimestamp_decoding 10 ok 1 - inode_test_xtimestamp_decoding 11 ok 1 - inode_test_xtimestamp_decoding 12 ok 1 - inode_test_xtimestamp_decoding 13 ok 1 - inode_test_xtimestamp_decoding 14 ok 1 - inode_test_xtimestamp_decoding 15 ok 1 - inode_test_xtimestamp_decoding 16 ok 1 - ext4_inode_test (followed by other kunit test outputs)
Hmm, interesting. Let me play with your patch a bit.
One option is to just have the test case number increment as well, i.e. have this: | ok 1 - inode_test_xtimestamp_decoding#1 | ok 2 - inode_test_xtimestamp_decoding#2 | ok 3 - inode_test_xtimestamp_decoding#3 | ok 4 - inode_test_xtimestamp_decoding#4 | ok 5 - inode_test_xtimestamp_decoding#5 ...
Or is there something else I missed?
Right, so TAP wants the exact number of tests it will run ahead of time. In which case we can still put the results of each parameterized test in a diagnostic. Please see my proposed patch below, which still does proper initialization/destruction of each parameter case as if it was its own test case.
With it the output looks as follows:
| TAP version 14 | 1..6 | # Subtest: ext4_inode_test | 1..1 | # ok param#0 - inode_test_xtimestamp_decoding | # ok param#1 - inode_test_xtimestamp_decoding | # ok param#2 - inode_test_xtimestamp_decoding | # ok param#3 - inode_test_xtimestamp_decoding | # ok param#4 - inode_test_xtimestamp_decoding | # ok param#5 - inode_test_xtimestamp_decoding | # ok param#6 - inode_test_xtimestamp_decoding | # ok param#7 - inode_test_xtimestamp_decoding | # ok param#8 - inode_test_xtimestamp_decoding | # ok param#9 - inode_test_xtimestamp_decoding | # ok param#10 - inode_test_xtimestamp_decoding | # ok param#11 - inode_test_xtimestamp_decoding | # ok param#12 - inode_test_xtimestamp_decoding | # ok param#13 - inode_test_xtimestamp_decoding | # ok param#14 - inode_test_xtimestamp_decoding | # ok param#15 - inode_test_xtimestamp_decoding | ok 1 - inode_test_xtimestamp_decoding | ok 1 - ext4_inode_test
Would that be reasonable? If so, feel free to take the patch and test/adjust as required.
I'm not sure on the best format -- is there is a recommended format for parameterized test result output? If not, I suppose we can put anything we like into the diagnostic.
I think this format of output should be fine for parameterized tests. But, this patch has the same issue as earlier. While, the tests run and this is the output that can be seen using dmesg, it still causes an issue on using the kunit tool. It gives a similar error:
[11:07:38] [ERROR] expected 7 test suites, but got -1 [11:07:38] [ERROR] expected_suite_index -1, but got 2 [11:07:38] [ERROR] got unexpected test suite: kunit-try-catch-test [ERROR] no tests run! [11:07:38] ============================================================ [11:07:38] Testing complete. 0 tests run. 0 failed. 0 crashed.
Thanks!
Thanks, -- Marco
------ >8 ------
From ccbf4e2e190a2d7c6a94a51c9b1fb3b9a940e578 Mon Sep 17 00:00:00 2001 From: Arpitha Raghunandan 98.arpi@gmail.com Date: Tue, 27 Oct 2020 23:16:30 +0530 Subject: [PATCH] kunit: Support for Parameterized Testing
Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com
Changes v4->v5:
- Update kernel-doc comments.
- Use const void* for generator return and prev value types.
- Add kernel-doc comment for KUNIT_ARRAY_PARAM.
- Rework parameterized test case execution strategy: each parameter is executed as if it was its own test case, with its own test initialization and cleanup (init and exit are called, etc.). However, we cannot add new test cases per TAP protocol once we have already started execution. Instead, log the result of each parameter run as a diagnostic comment.
Changes v3->v4:
- Rename kunit variables
- Rename generator function helper macro
- Add documentation for generator approach
- Display test case name in case of failure along with param index
Changes v2->v3:
- Modifictaion of generator macro and method
Changes v1->v2:
- Use of a generator method to access test case parameters
include/kunit/test.h | 36 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 46 +++++++++++++++++++++++++++++++------------- 2 files changed, 69 insertions(+), 13 deletions(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ae5488a37e48 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,7 @@ struct kunit;
- @run_case: the function representing the actual test case.
- @name: the name of the test case.
- @generate_params: the generator function for parameterized tests.
- A test case is a function with the signature,
- ``void (*)(struct kunit *)``
@@ -141,6 +142,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name;
- const void* (*generate_params)(const void *prev);
/* private: internal use only. */ bool success; @@ -163,6 +165,22 @@ static inline char *kunit_status_to_string(bool status) */ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name } +/**
- KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case
- @test_name: a reference to a test case function.
- @gen_params: a reference to a parameter generator function.
- The generator function ``const void* gen_params(const void *prev)`` is used
- to lazily generate a series of arbitrarily typed values that fit into a
- void*. The argument @prev is the previously returned value, which should be
- used to derive the next value; @prev is set to NULL on the initial generator
- call. When no more values are available, the generator must return NULL.
- */
+#define KUNIT_CASE_PARAM(test_name, gen_params) \
{ .run_case = test_name, .name = #test_name, \
.generate_params = gen_params }
/**
- struct kunit_suite - describes a related collection of &struct kunit_case
@@ -208,6 +226,10 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch;
- /* param_value is the current parameter value for a test case. */
- const void *param_value;
- /* param_index stores the index of the parameter in parameterized tests. */
- int param_index; /*
- success starts as true, and may only be set to false during a
- test case; thus, it is safe to update this across multiple
@@ -1742,4 +1764,18 @@ do { \ fmt, \ ##__VA_ARGS__) +/**
- KUNIT_ARRAY_PARAM() - Define test parameter generator from an array.
- @name: prefix for the test parameter generator function.
- @array: array of test parameters.
- Define function @name_gen_params which uses @array to generate parameters.
- */
+#define KUNIT_ARRAY_PARAM(name, array) \
- static const void *name##_gen_params(const void *prev) \
- { \
typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \
return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \
- }
#endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..453ebe4da77d 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -325,29 +325,25 @@ static void kunit_catch_run_case(void *data)
- occur in a test case and reports them as failures.
*/ static void kunit_run_case_catch_errors(struct kunit_suite *suite,
struct kunit_case *test_case)
struct kunit_case *test_case,
struct kunit *test)
{ struct kunit_try_catch_context context; struct kunit_try_catch *try_catch;
- struct kunit test;
- kunit_init_test(&test, test_case->name, test_case->log);
- try_catch = &test.try_catch;
- kunit_init_test(test, test_case->name, test_case->log);
- try_catch = &test->try_catch;
kunit_try_catch_init(try_catch,
&test,
test, kunit_try_run_case, kunit_catch_run_case);
- context.test = &test;
- context.test = test; context.suite = suite; context.test_case = test_case; kunit_try_catch_run(try_catch, &context);
- test_case->success = test.success;
- kunit_print_ok_not_ok(&test, true, test_case->success,
kunit_test_case_num(suite, test_case),
test_case->name);
- test_case->success = test->success;
} int kunit_run_tests(struct kunit_suite *suite) @@ -356,8 +352,32 @@ int kunit_run_tests(struct kunit_suite *suite) kunit_print_subtest_start(suite);
- kunit_suite_for_each_test_case(suite, test_case)
kunit_run_case_catch_errors(suite, test_case);
- kunit_suite_for_each_test_case(suite, test_case) {
struct kunit test = { .param_value = NULL, .param_index = 0 };
bool test_success = true;
if (test_case->generate_params)
test.param_value = test_case->generate_params(NULL);
do {
kunit_run_case_catch_errors(suite, test_case, &test);
test_success &= test_case->success;
if (test_case->generate_params) {
kunit_log(KERN_INFO, &test,
KUNIT_SUBTEST_INDENT
"# %s param#%d - %s",
kunit_status_to_string(test.success),
test.param_index, test_case->name);
test.param_value = test_case->generate_params(test.param_value);
test.param_index++;
}
} while (test.param_value);
kunit_print_ok_not_ok(&test, true, test_success,
kunit_test_case_num(suite, test_case),
test_case->name);
- }
kunit_print_subtest_end(suite);
On Fri, 6 Nov 2020 at 06:54, Arpitha Raghunandan 98.arpi@gmail.com wrote:
On 06/11/20 1:25 am, Marco Elver wrote:
On Thu, Nov 05, 2020 at 04:02PM +0100, Marco Elver wrote:
On Thu, 5 Nov 2020 at 15:30, Arpitha Raghunandan 98.arpi@gmail.com wrote:
[...]
I tried adding support to run each parameter as a distinct test case by making changes to kunit_run_case_catch_errors(). The issue here is that since the results are displayed in KTAP format, this change will result in each parameter being considered a subtest of another subtest (test case in KUnit).
Do you have example output? That might help understand what's going on.
The change that I tried can be seen here (based on the v4 patch): https://gist.github.com/arpi-r/4822899087ca4cc34572ed9e45cc5fee.
Using the kunit tool, I get this error:
[19:20:41] [ERROR] expected 7 test suites, but got -1 [ERROR] no tests run! [19:20:41] ============================================================ [19:20:41] Testing complete. 0 tests run. 0 failed. 0 crashed.
But this error is only because of how the tool displays the results. The test actually does run, as can be seen in the dmesg output:
TAP version 14 1..7 # Subtest: ext4_inode_test 1..1 ok 1 - inode_test_xtimestamp_decoding 1 ok 1 - inode_test_xtimestamp_decoding 2 ok 1 - inode_test_xtimestamp_decoding 3 ok 1 - inode_test_xtimestamp_decoding 4 ok 1 - inode_test_xtimestamp_decoding 5 ok 1 - inode_test_xtimestamp_decoding 6 ok 1 - inode_test_xtimestamp_decoding 7 ok 1 - inode_test_xtimestamp_decoding 8 ok 1 - inode_test_xtimestamp_decoding 9 ok 1 - inode_test_xtimestamp_decoding 10 ok 1 - inode_test_xtimestamp_decoding 11 ok 1 - inode_test_xtimestamp_decoding 12 ok 1 - inode_test_xtimestamp_decoding 13 ok 1 - inode_test_xtimestamp_decoding 14 ok 1 - inode_test_xtimestamp_decoding 15 ok 1 - inode_test_xtimestamp_decoding 16 ok 1 - ext4_inode_test (followed by other kunit test outputs)
Hmm, interesting. Let me play with your patch a bit.
One option is to just have the test case number increment as well, i.e. have this: | ok 1 - inode_test_xtimestamp_decoding#1 | ok 2 - inode_test_xtimestamp_decoding#2 | ok 3 - inode_test_xtimestamp_decoding#3 | ok 4 - inode_test_xtimestamp_decoding#4 | ok 5 - inode_test_xtimestamp_decoding#5 ...
Or is there something else I missed?
Right, so TAP wants the exact number of tests it will run ahead of time. In which case we can still put the results of each parameterized test in a diagnostic. Please see my proposed patch below, which still does proper initialization/destruction of each parameter case as if it was its own test case.
With it the output looks as follows:
| TAP version 14 | 1..6 | # Subtest: ext4_inode_test | 1..1 | # ok param#0 - inode_test_xtimestamp_decoding | # ok param#1 - inode_test_xtimestamp_decoding | # ok param#2 - inode_test_xtimestamp_decoding | # ok param#3 - inode_test_xtimestamp_decoding | # ok param#4 - inode_test_xtimestamp_decoding | # ok param#5 - inode_test_xtimestamp_decoding | # ok param#6 - inode_test_xtimestamp_decoding | # ok param#7 - inode_test_xtimestamp_decoding | # ok param#8 - inode_test_xtimestamp_decoding | # ok param#9 - inode_test_xtimestamp_decoding | # ok param#10 - inode_test_xtimestamp_decoding | # ok param#11 - inode_test_xtimestamp_decoding | # ok param#12 - inode_test_xtimestamp_decoding | # ok param#13 - inode_test_xtimestamp_decoding | # ok param#14 - inode_test_xtimestamp_decoding | # ok param#15 - inode_test_xtimestamp_decoding | ok 1 - inode_test_xtimestamp_decoding | ok 1 - ext4_inode_test
Would that be reasonable? If so, feel free to take the patch and test/adjust as required.
I'm not sure on the best format -- is there is a recommended format for parameterized test result output? If not, I suppose we can put anything we like into the diagnostic.
I think this format of output should be fine for parameterized tests. But, this patch has the same issue as earlier. While, the tests run and this is the output that can be seen using dmesg, it still causes an issue on using the kunit tool. It gives a similar error:
[11:07:38] [ERROR] expected 7 test suites, but got -1 [11:07:38] [ERROR] expected_suite_index -1, but got 2 [11:07:38] [ERROR] got unexpected test suite: kunit-try-catch-test [ERROR] no tests run! [11:07:38] ============================================================ [11:07:38] Testing complete. 0 tests run. 0 failed. 0 crashed.
I'd suggest testing without these patches and diffing the output. AFAIK we're not adding any new non-# output, so it might be a pre-existing bug in some parsing code. Either that, or the parsing code does not respect the # correctly?
Thanks, -- Marco
On Fri, Nov 06, 2020 at 09:11AM +0100, Marco Elver wrote:
On Fri, 6 Nov 2020 at 06:54, Arpitha Raghunandan 98.arpi@gmail.com wrote:
[...]
I think this format of output should be fine for parameterized tests. But, this patch has the same issue as earlier. While, the tests run and this is the output that can be seen using dmesg, it still causes an issue on using the kunit tool. It gives a similar error:
[11:07:38] [ERROR] expected 7 test suites, but got -1 [11:07:38] [ERROR] expected_suite_index -1, but got 2 [11:07:38] [ERROR] got unexpected test suite: kunit-try-catch-test [ERROR] no tests run! [11:07:38] ============================================================ [11:07:38] Testing complete. 0 tests run. 0 failed. 0 crashed.
I'd suggest testing without these patches and diffing the output. AFAIK we're not adding any new non-# output, so it might be a pre-existing bug in some parsing code. Either that, or the parsing code does not respect the # correctly?
Hmm, tools/testing/kunit/kunit_parser.py has
SUBTEST_DIAGNOSTIC = re.compile(r'^[\s]+# .*?: (.*)$')
, which seems to expect a ': ' in there. Not sure if this is required by TAP, but let's leave this alone for now.
We can change the output to not trip this up, which might also be more a consistent diagnostic format vs. other diagnostics. See the revised patch below. With it the output is as follows:
| TAP version 14 | 1..6 | # Subtest: ext4_inode_test | 1..1 | # inode_test_xtimestamp_decoding: param-0 ok | # inode_test_xtimestamp_decoding: param-1 ok | # inode_test_xtimestamp_decoding: param-2 ok | # inode_test_xtimestamp_decoding: param-3 ok | # inode_test_xtimestamp_decoding: param-4 ok | # inode_test_xtimestamp_decoding: param-5 ok | # inode_test_xtimestamp_decoding: param-6 ok | # inode_test_xtimestamp_decoding: param-7 ok | # inode_test_xtimestamp_decoding: param-8 ok | # inode_test_xtimestamp_decoding: param-9 ok | # inode_test_xtimestamp_decoding: param-10 ok | # inode_test_xtimestamp_decoding: param-11 ok | # inode_test_xtimestamp_decoding: param-12 ok | # inode_test_xtimestamp_decoding: param-13 ok | # inode_test_xtimestamp_decoding: param-14 ok | # inode_test_xtimestamp_decoding: param-15 ok | ok 1 - inode_test_xtimestamp_decoding | ok 1 - ext4_inode_test
And kunit-tool seems to be happy as well.
Thanks, -- Marco
------ >8 ------
From 13a94d75d6b1b430e89fcff2cd76629b56b9d636 Mon Sep 17 00:00:00 2001 From: Arpitha Raghunandan 98.arpi@gmail.com Date: Tue, 27 Oct 2020 23:16:30 +0530 Subject: [PATCH] kunit: Support for Parameterized Testing
Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com --- Changes v4->v5: - Update kernel-doc comments. - Use const void* for generator return and prev value types. - Add kernel-doc comment for KUNIT_ARRAY_PARAM. - Rework parameterized test case execution strategy: each parameter is executed as if it was its own test case, with its own test initialization and cleanup (init and exit are called, etc.). However, we cannot add new test cases per TAP protocol once we have already started execution. Instead, log the result of each parameter run as a diagnostic comment. Changes v3->v4: - Rename kunit variables - Rename generator function helper macro - Add documentation for generator approach - Display test case name in case of failure along with param index Changes v2->v3: - Modifictaion of generator macro and method Changes v1->v2: - Use of a generator method to access test case parameters --- include/kunit/test.h | 36 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 46 +++++++++++++++++++++++++++++++------------- 2 files changed, 69 insertions(+), 13 deletions(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ae5488a37e48 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,7 @@ struct kunit; * * @run_case: the function representing the actual test case. * @name: the name of the test case. + * @generate_params: the generator function for parameterized tests. * * A test case is a function with the signature, * ``void (*)(struct kunit *)`` @@ -141,6 +142,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name; + const void* (*generate_params)(const void *prev);
/* private: internal use only. */ bool success; @@ -163,6 +165,22 @@ static inline char *kunit_status_to_string(bool status) */ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
+/** + * KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case + * + * @test_name: a reference to a test case function. + * @gen_params: a reference to a parameter generator function. + * + * The generator function ``const void* gen_params(const void *prev)`` is used + * to lazily generate a series of arbitrarily typed values that fit into a + * void*. The argument @prev is the previously returned value, which should be + * used to derive the next value; @prev is set to NULL on the initial generator + * call. When no more values are available, the generator must return NULL. + */ +#define KUNIT_CASE_PARAM(test_name, gen_params) \ + { .run_case = test_name, .name = #test_name, \ + .generate_params = gen_params } + /** * struct kunit_suite - describes a related collection of &struct kunit_case * @@ -208,6 +226,10 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch; + /* param_value is the current parameter value for a test case. */ + const void *param_value; + /* param_index stores the index of the parameter in parameterized tests. */ + int param_index; /* * success starts as true, and may only be set to false during a * test case; thus, it is safe to update this across multiple @@ -1742,4 +1764,18 @@ do { \ fmt, \ ##__VA_ARGS__)
+/** + * KUNIT_ARRAY_PARAM() - Define test parameter generator from an array. + * @name: prefix for the test parameter generator function. + * @array: array of test parameters. + * + * Define function @name_gen_params which uses @array to generate parameters. + */ +#define KUNIT_ARRAY_PARAM(name, array) \ + static const void *name##_gen_params(const void *prev) \ + { \ + typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \ + return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \ + } + #endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..329fee9e0634 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -325,29 +325,25 @@ static void kunit_catch_run_case(void *data) * occur in a test case and reports them as failures. */ static void kunit_run_case_catch_errors(struct kunit_suite *suite, - struct kunit_case *test_case) + struct kunit_case *test_case, + struct kunit *test) { struct kunit_try_catch_context context; struct kunit_try_catch *try_catch; - struct kunit test;
- kunit_init_test(&test, test_case->name, test_case->log); - try_catch = &test.try_catch; + kunit_init_test(test, test_case->name, test_case->log); + try_catch = &test->try_catch;
kunit_try_catch_init(try_catch, - &test, + test, kunit_try_run_case, kunit_catch_run_case); - context.test = &test; + context.test = test; context.suite = suite; context.test_case = test_case; kunit_try_catch_run(try_catch, &context);
- test_case->success = test.success; - - kunit_print_ok_not_ok(&test, true, test_case->success, - kunit_test_case_num(suite, test_case), - test_case->name); + test_case->success = test->success; }
int kunit_run_tests(struct kunit_suite *suite) @@ -356,8 +352,32 @@ int kunit_run_tests(struct kunit_suite *suite)
kunit_print_subtest_start(suite);
- kunit_suite_for_each_test_case(suite, test_case) - kunit_run_case_catch_errors(suite, test_case); + kunit_suite_for_each_test_case(suite, test_case) { + struct kunit test = { .param_value = NULL, .param_index = 0 }; + bool test_success = true; + + if (test_case->generate_params) + test.param_value = test_case->generate_params(NULL); + + do { + kunit_run_case_catch_errors(suite, test_case, &test); + test_success &= test_case->success; + + if (test_case->generate_params) { + kunit_log(KERN_INFO, &test, + KUNIT_SUBTEST_INDENT + "# %s: param-%d %s", + test_case->name, test.param_index, + kunit_status_to_string(test.success)); + test.param_value = test_case->generate_params(test.param_value); + test.param_index++; + } + } while (test.param_value); + + kunit_print_ok_not_ok(&test, true, test_success, + kunit_test_case_num(suite, test_case), + test_case->name); + }
kunit_print_subtest_end(suite);
On 06/11/20 6:04 pm, Marco Elver wrote:
On Fri, Nov 06, 2020 at 09:11AM +0100, Marco Elver wrote:
On Fri, 6 Nov 2020 at 06:54, Arpitha Raghunandan 98.arpi@gmail.com wrote:
[...]
I think this format of output should be fine for parameterized tests. But, this patch has the same issue as earlier. While, the tests run and this is the output that can be seen using dmesg, it still causes an issue on using the kunit tool. It gives a similar error:
[11:07:38] [ERROR] expected 7 test suites, but got -1 [11:07:38] [ERROR] expected_suite_index -1, but got 2 [11:07:38] [ERROR] got unexpected test suite: kunit-try-catch-test [ERROR] no tests run! [11:07:38] ============================================================ [11:07:38] Testing complete. 0 tests run. 0 failed. 0 crashed.
I'd suggest testing without these patches and diffing the output. AFAIK we're not adding any new non-# output, so it might be a pre-existing bug in some parsing code. Either that, or the parsing code does not respect the # correctly?
Hmm, tools/testing/kunit/kunit_parser.py has
SUBTEST_DIAGNOSTIC = re.compile(r'^[\s]+# .*?: (.*)$')
, which seems to expect a ': ' in there. Not sure if this is required by TAP, but let's leave this alone for now.
Oh okay.
We can change the output to not trip this up, which might also be more a consistent diagnostic format vs. other diagnostics. See the revised patch below. With it the output is as follows:
| TAP version 14 | 1..6 | # Subtest: ext4_inode_test | 1..1 | # inode_test_xtimestamp_decoding: param-0 ok | # inode_test_xtimestamp_decoding: param-1 ok | # inode_test_xtimestamp_decoding: param-2 ok | # inode_test_xtimestamp_decoding: param-3 ok | # inode_test_xtimestamp_decoding: param-4 ok | # inode_test_xtimestamp_decoding: param-5 ok | # inode_test_xtimestamp_decoding: param-6 ok | # inode_test_xtimestamp_decoding: param-7 ok | # inode_test_xtimestamp_decoding: param-8 ok | # inode_test_xtimestamp_decoding: param-9 ok | # inode_test_xtimestamp_decoding: param-10 ok | # inode_test_xtimestamp_decoding: param-11 ok | # inode_test_xtimestamp_decoding: param-12 ok | # inode_test_xtimestamp_decoding: param-13 ok | # inode_test_xtimestamp_decoding: param-14 ok | # inode_test_xtimestamp_decoding: param-15 ok | ok 1 - inode_test_xtimestamp_decoding | ok 1 - ext4_inode_test
And kunit-tool seems to be happy as well.
Yes this works as expected. Thank you! I will send this patch as v5.
Thanks, -- Marco
------ >8 ------
From 13a94d75d6b1b430e89fcff2cd76629b56b9d636 Mon Sep 17 00:00:00 2001 From: Arpitha Raghunandan 98.arpi@gmail.com Date: Tue, 27 Oct 2020 23:16:30 +0530 Subject: [PATCH] kunit: Support for Parameterized Testing
Implementation of support for parameterized testing in KUnit. This approach requires the creation of a test case using the KUNIT_CASE_PARAM macro that accepts a generator function as input. This generator function should return the next parameter given the previous parameter in parameterized tests. It also provides a macro to generate common-case generators.
Signed-off-by: Arpitha Raghunandan 98.arpi@gmail.com Co-developed-by: Marco Elver elver@google.com Signed-off-by: Marco Elver elver@google.com
Changes v4->v5:
- Update kernel-doc comments.
- Use const void* for generator return and prev value types.
- Add kernel-doc comment for KUNIT_ARRAY_PARAM.
- Rework parameterized test case execution strategy: each parameter is executed as if it was its own test case, with its own test initialization and cleanup (init and exit are called, etc.). However, we cannot add new test cases per TAP protocol once we have already started execution. Instead, log the result of each parameter run as a diagnostic comment.
Changes v3->v4:
- Rename kunit variables
- Rename generator function helper macro
- Add documentation for generator approach
- Display test case name in case of failure along with param index
Changes v2->v3:
- Modifictaion of generator macro and method
Changes v1->v2:
- Use of a generator method to access test case parameters
include/kunit/test.h | 36 ++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 46 +++++++++++++++++++++++++++++++------------- 2 files changed, 69 insertions(+), 13 deletions(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h index 9197da792336..ae5488a37e48 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -107,6 +107,7 @@ struct kunit;
- @run_case: the function representing the actual test case.
- @name: the name of the test case.
- @generate_params: the generator function for parameterized tests.
- A test case is a function with the signature,
- ``void (*)(struct kunit *)``
@@ -141,6 +142,7 @@ struct kunit; struct kunit_case { void (*run_case)(struct kunit *test); const char *name;
- const void* (*generate_params)(const void *prev);
/* private: internal use only. */ bool success; @@ -163,6 +165,22 @@ static inline char *kunit_status_to_string(bool status) */ #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name } +/**
- KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case
- @test_name: a reference to a test case function.
- @gen_params: a reference to a parameter generator function.
- The generator function ``const void* gen_params(const void *prev)`` is used
- to lazily generate a series of arbitrarily typed values that fit into a
- void*. The argument @prev is the previously returned value, which should be
- used to derive the next value; @prev is set to NULL on the initial generator
- call. When no more values are available, the generator must return NULL.
- */
+#define KUNIT_CASE_PARAM(test_name, gen_params) \
{ .run_case = test_name, .name = #test_name, \
.generate_params = gen_params }
/**
- struct kunit_suite - describes a related collection of &struct kunit_case
@@ -208,6 +226,10 @@ struct kunit { const char *name; /* Read only after initialization! */ char *log; /* Points at case log after initialization */ struct kunit_try_catch try_catch;
- /* param_value is the current parameter value for a test case. */
- const void *param_value;
- /* param_index stores the index of the parameter in parameterized tests. */
- int param_index; /*
- success starts as true, and may only be set to false during a
- test case; thus, it is safe to update this across multiple
@@ -1742,4 +1764,18 @@ do { \ fmt, \ ##__VA_ARGS__) +/**
- KUNIT_ARRAY_PARAM() - Define test parameter generator from an array.
- @name: prefix for the test parameter generator function.
- @array: array of test parameters.
- Define function @name_gen_params which uses @array to generate parameters.
- */
+#define KUNIT_ARRAY_PARAM(name, array) \
- static const void *name##_gen_params(const void *prev) \
- { \
typeof((array)[0]) * __next = prev ? ((typeof(__next)) prev) + 1 : (array); \
return __next - (array) < ARRAY_SIZE((array)) ? __next : NULL; \
- }
#endif /* _KUNIT_TEST_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 750704abe89a..329fee9e0634 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -325,29 +325,25 @@ static void kunit_catch_run_case(void *data)
- occur in a test case and reports them as failures.
*/ static void kunit_run_case_catch_errors(struct kunit_suite *suite,
struct kunit_case *test_case)
struct kunit_case *test_case,
struct kunit *test)
{ struct kunit_try_catch_context context; struct kunit_try_catch *try_catch;
- struct kunit test;
- kunit_init_test(&test, test_case->name, test_case->log);
- try_catch = &test.try_catch;
- kunit_init_test(test, test_case->name, test_case->log);
- try_catch = &test->try_catch;
kunit_try_catch_init(try_catch,
&test,
test, kunit_try_run_case, kunit_catch_run_case);
- context.test = &test;
- context.test = test; context.suite = suite; context.test_case = test_case; kunit_try_catch_run(try_catch, &context);
- test_case->success = test.success;
- kunit_print_ok_not_ok(&test, true, test_case->success,
kunit_test_case_num(suite, test_case),
test_case->name);
- test_case->success = test->success;
} int kunit_run_tests(struct kunit_suite *suite) @@ -356,8 +352,32 @@ int kunit_run_tests(struct kunit_suite *suite) kunit_print_subtest_start(suite);
- kunit_suite_for_each_test_case(suite, test_case)
kunit_run_case_catch_errors(suite, test_case);
- kunit_suite_for_each_test_case(suite, test_case) {
struct kunit test = { .param_value = NULL, .param_index = 0 };
bool test_success = true;
if (test_case->generate_params)
test.param_value = test_case->generate_params(NULL);
do {
kunit_run_case_catch_errors(suite, test_case, &test);
test_success &= test_case->success;
if (test_case->generate_params) {
kunit_log(KERN_INFO, &test,
KUNIT_SUBTEST_INDENT
"# %s: param-%d %s",
test_case->name, test.param_index,
kunit_status_to_string(test.success));
test.param_value = test_case->generate_params(test.param_value);
test.param_index++;
}
} while (test.param_value);
kunit_print_ok_not_ok(&test, true, test_success,
kunit_test_case_num(suite, test_case),
test_case->name);
- }
kunit_print_subtest_end(suite);
linux-kselftest-mirror@lists.linaro.org