From 1d69c45f24dcd514e6c36901a58ddc5792774e27 Mon Sep 17 00:00:00 2001 From: Alvaro Garcia Date: Tue, 8 Feb 2022 13:07:37 +0400 Subject: [PATCH 1/3] [83] Ignore idea, cmake folders, actual log --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 115d637..f192d3d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,9 @@ /example /*.exe /*.dll + +.idea/ +cmake-*/ + +*_actual.log + From 1381254c8d4cd3e0958eda1ebf2e2561aac8eb26 Mon Sep 17 00:00:00 2001 From: Alvaro Garcia Date: Tue, 8 Feb 2022 13:08:12 +0400 Subject: [PATCH 2/3] [83] Add the test_setup scenario --- Makefile | 3 + test_setup.c | 305 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 308 insertions(+) create mode 100644 test_setup.c diff --git a/Makefile b/Makefile index 65d33fc..dc1ac54 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,9 @@ endif example$(EXTENSION): munit.h munit.c example.c $(CC) $(CFLAGS) -o $@ munit.c example.c +test_setup$(EXTENSION): munit.h munit.c test_setup.c + $(CC) $(CFLAGS) -o $@ munit.c test_setup.c + test: $(TEST_ENV) ./example$(EXTENSION) diff --git a/test_setup.c b/test_setup.c new file mode 100644 index 0000000..f1d0d32 --- /dev/null +++ b/test_setup.c @@ -0,0 +1,305 @@ +/* Example file for using µnit. + * + * µnit is MIT-licensed, but for this file and this file alone: + * + * To the extent possible under law, the author(s) of this file have + * waived all copyright and related or neighboring rights to this + * work. See for + * details. + *********************************************************************/ + +#include "munit.h" + +/* This is just to disable an MSVC warning about conditional + * expressions being constant, which you shouldn't have to do for your + * code. It's only here because we want to be able to do silly things + * like assert that 0 != 1 for our demo. */ +#if defined(_MSC_VER) +#pragma warning(disable : 4127) +#endif + +/* Tests are functions that return void, and take a single void* + * parameter. We'll get to what that parameter is later. */ +static MunitResult +test_compare(const MunitParameter params[], void* data) { + /* We'll use these later */ + const unsigned char val_uchar = 'b'; + const short val_short = 1729; + double pi = 3.141592654; + char* stewardesses = "stewardesses"; + char* most_fun_word_to_type; + + /* These are just to silence compiler warnings about the parameters + * being unused. */ + (void) params; + (void) data; + + /* Let's start with the basics. */ + munit_assert(0 != 1); + + /* There is also the more verbose, though slightly more descriptive + munit_assert_true/false: */ + munit_assert_false(0); + + /* You can also call munit_error and munit_errorf yourself. We + * won't do it is used to indicate a failure, but here is what it + * would look like: */ + /* munit_error("FAIL"); */ + /* munit_errorf("Goodbye, cruel %s", "world"); */ + + /* There are macros for comparing lots of types. */ + munit_assert_char('a', ==, 'a'); +// munit_assert_char('a', ==, 'b'); + + /* Sure, you could just assert('a' == 'a'), but if you did that, a + * failed assertion would just say something like "assertion failed: + * val_uchar == 'b'". µnit will tell you the actual values, so a + * failure here would result in something like "assertion failed: + * val_uchar == 'b' ('X' == 'b')." */ + munit_assert_uchar(val_uchar, ==, 'b'); + + /* Obviously we can handle values larger than 'char' and 'uchar'. + * There are versions for char, short, int, long, long long, + * int8/16/32/64_t, as well as the unsigned versions of them all. */ + munit_assert_short(42, <, val_short); + + /* There is also support for size_t. + * + * The longest word in English without repeating any letters is + * "uncopyrightables", which has uncopyrightable (and + * dermatoglyphics, which is the study of fingerprints) beat by a + * character */ + munit_assert_size(strlen("uncopyrightables"), >, strlen("dermatoglyphics")); + + /* Of course there is also support for doubles and floats. */ + munit_assert_double(pi, ==, 3.141592654); + + /* If you want to compare two doubles for equality, you might want + * to consider using munit_assert_double_equal. It compares two + * doubles for equality within a precison of 1.0 x 10^-(precision). + * Note that precision (the third argument to the macro) needs to be + * fully evaluated to an integer by the preprocessor so µnit doesn't + * have to depend pow, which is often in libm not libc. */ + munit_assert_double_equal(3.141592654, 3.141592653589793, 9); + + /* And if you want to check strings for equality (or inequality), + * there is munit_assert_string_equal/not_equal. + * + * "stewardesses" is the longest word you can type on a QWERTY + * keyboard with only one hand, which makes it loads of fun to type. + * If I'm going to have to type a string repeatedly, let's make it a + * good one! */ + munit_assert_string_equal(stewardesses, "stewardesses"); + + /* A personal favorite macro which is fantastic if you're working + * with binary data, is the one which naïvely checks two blobs of + * memory for equality. If this fails it will tell you the offset + * of the first differing byte. */ + munit_assert_memory_equal(7, stewardesses, "steward"); + + /* You can also make sure that two blobs differ *somewhere*: */ + munit_assert_memory_not_equal(8, stewardesses, "steward"); + + /* There are equal/not_equal macros for pointers, too: */ + most_fun_word_to_type = stewardesses; + munit_assert_ptr_equal(most_fun_word_to_type, stewardesses); + + /* And null/not_null */ + munit_assert_null(NULL); + munit_assert_not_null(most_fun_word_to_type); + + /* Lets verify that the data parameter is what we expected. We'll + * see where this comes from in a bit. + * + * Note that the casting isn't usually required; if you give this + * function a real pointer (instead of a number like 0xdeadbeef) it + * would work as expected. */ + munit_assert_ptr_equal(data, (void*) (uintptr_t) 0xdeadbeef); + + return MUNIT_OK; +} + +static MunitResult +test_rand(const MunitParameter params[], void* user_data) { + int random_int; + double random_dbl; + munit_uint8_t data[5]; + + (void) params; + (void) user_data; + + /* One thing missing from a lot of unit testing frameworks is a + * random number generator. You can't just use srand/rand because + * the implementation varies across different platforms, and it's + * important to be able to look at the seed used in a failing test + * to see if you can reproduce it. Some randomness is a fantastic + * thing to have in your tests, I don't know why more people don't + * do it... + * + * µnit's PRNG is re-seeded with the same value for each iteration + * of each test. The seed is retrieved from the MUNIT_SEED + * envirnment variable or, if none is provided, one will be + * (pseudo-)randomly generated. */ + + /* If you need an integer in a given range */ + random_int = munit_rand_int_range(128, 4096); + munit_assert_int(random_int, >=, 128); + munit_assert_int(random_int, <=, 4096); + + /* Or maybe you want a double, between 0 and 1: */ + random_dbl = munit_rand_double(); + munit_assert_double(random_dbl, >=, 0.0); + munit_assert_double(random_dbl, <=, 1.0); + + /* Of course, you want to be able to reproduce bugs discovered + * during testing, so every time the tests are run they print the + * random seed used. When you want to reproduce a result, just put + * that random seed in the MUNIT_SEED environment variable; it even + * works on different platforms. + * + * If you want this to pass, use 0xdeadbeef as the random seed and + * uncomment the next line of code. Note that the PRNG is not + * re-seeded between iterations of the same test, so this will only + * work on the first iteration. */ + /* munit_assert_uint32(munit_rand_uint32(), ==, 1306447409); */ + + /* You can also get blobs of random memory: */ + munit_rand_memory(sizeof(data), data); + + return MUNIT_OK; +} + +/* The setup function, if you provide one, for a test will be run + * before the test, and the return value will be passed as the sole + * parameter to the test function. */ +static void* +test_compare_setup(const MunitParameter params[], void* user_data) { + (void) params; + + munit_assert_string_equal(user_data, "µnit"); + return (void*) (uintptr_t) 0xdeadbeef; +} + +/* To clean up after a test, you can use a tear down function. The + * fixture argument is the value returned by the setup function + * above. */ +static void +test_compare_tear_down(void* fixture) { + munit_assert_ptr_equal(fixture, (void*) (uintptr_t) 0xdeadbeef); +} + +static char* foo_params[] = { + (char*) "one", (char*) "two", (char*) "three", NULL}; + +static char* bar_params[] = { + (char*) "red", (char*) "green", (char*) "blue", NULL}; + +static MunitParameterEnum test_params[] = { + {(char*) "foo", foo_params}, + {(char*) "bar", bar_params}, + {(char*) "baz", NULL}, + {NULL, NULL}, +}; + +/* Creating a test suite is pretty simple. First, you'll need an + * array of tests: */ +static MunitTest test_suite_tests[] = { + {/* The name is just a unique human-readable way to identify the + * test. You can use it to run a specific test if you want, but + * usually it's mostly decorative. */ + (char*) "/compare", + /* You probably won't be surprised to learn that the tests are + * functions. */ + test_compare, + /* If you want, you can supply a function to set up a fixture. If + * you supply NULL, the user_data parameter from munit_suite_main + * will be used directly. If, however, you provide a callback + * here the user_data parameter will be passed to this callback, + * and the return value from this callback will be passed to the + * test function. + * + * For our example we don't really need a fixture, but lets + * provide one anyways. */ + test_compare_setup, + /* If you passed a callback for the fixture setup function, you + * may want to pass a corresponding callback here to reverse the + * operation. */ + test_compare_tear_down, + /* Finally, there is a bitmask for options you can pass here. You + * can provide either MUNIT_TEST_OPTION_NONE or 0 here to use the + * defaults. */ + MUNIT_TEST_OPTION_NONE, + NULL}, + /* Usually this is written in a much more compact format; all these + * comments kind of ruin that, though. Here is how you'll usually + * see entries written: */ + {(char*) "/rand", test_rand, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL}, + /* To tell the test runner when the array is over, just add a NULL + * entry at the end. */ + // { (char*) "/example/parameters", test_parameters, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL }, + {NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL}}; +static MunitTest test_suite_tests2[] = { + {(char*) "/compare", test_compare, test_compare_setup, test_compare_tear_down, MUNIT_TEST_OPTION_NONE, NULL}, + {(char*) "/rand", test_rand, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL}, + {NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL}}; + +static const MunitSuite other_suites_level3_sym[] = { + {"/sha2", test_suite_tests, NULL, 1, MUNIT_SUITE_OPTION_NONE}, + {"/sha3", test_suite_tests2, NULL, 1, MUNIT_SUITE_OPTION_NONE}, + {NULL, NULL, NULL, 0, MUNIT_SUITE_OPTION_NONE}}; + +static const MunitSuite other_suites_level3_asym[] = { + {"/x25519", test_suite_tests, NULL, 1, MUNIT_SUITE_OPTION_NONE}, + {"/NIST", test_suite_tests2, NULL, 1, MUNIT_SUITE_OPTION_NONE}, + {NULL, NULL, NULL, 0, MUNIT_SUITE_OPTION_NONE}}; + +static const MunitSuite other_suites_level2[] = { + {"/symmetric", NULL, other_suites_level3_sym, 1, MUNIT_SUITE_OPTION_NONE}, + {"/asymmetric", NULL, other_suites_level3_asym, 1, MUNIT_SUITE_OPTION_NONE}, + {NULL, NULL, NULL, 0, MUNIT_SUITE_OPTION_NONE}}; + +static const MunitSuite other_suites[] = { + {"/kat", NULL, other_suites_level2, 1, MUNIT_SUITE_OPTION_NONE}, + {"/unit", NULL, other_suites_level2, 1, MUNIT_SUITE_OPTION_NONE}, + {"/perf", NULL, other_suites_level2, 1, MUNIT_SUITE_OPTION_NONE}, + {NULL, NULL, NULL, 0, MUNIT_SUITE_OPTION_NONE}}; + +/* Now we'll actually declare the test suite. You could do this in + * the main function, or on the heap, or whatever you want. */ +static const MunitSuite test_suite = { + /* This string will be prepended to all test names in this suite; + * for example, "/example/rand" will become "/µnit/example/rand". + * Note that, while it doesn't really matter for the top-level + * suite, NULL signal the end of an array of tests; you should use + * an empty string ("") instead. */ + (char*) "", + /* The first parameter is the array of test suites. */ + NULL, + /* In addition to containing test cases, suites can contain other + * test suites. This isn't necessary in this example, but it can be + * a great help to projects with lots of tests by making it easier + * to spread the tests across many files. This is where you would + * put "other_suites" (which is commented out above). */ + other_suites, + /* An interesting feature of µnit is that it supports automatically + * running multiple iterations of the tests. This is usually only + * interesting if you make use of the PRNG to randomize your tests + * cases a bit, or if you are doing performance testing and want to + * average multiple runs. 0 is an alias for 1. */ + 1, + /* Just like MUNIT_TEST_OPTION_NONE, you can provide + * MUNIT_SUITE_OPTION_NONE or 0 to use the default settings. */ + MUNIT_SUITE_OPTION_NONE}; + +/* This is only necessary for EXIT_SUCCESS and EXIT_FAILURE, which you + * *should* be using but probably aren't (no, zero and non-zero don't + * always mean success and failure). I guess my point is that nothing + * about µnit requires it. */ +#include + +int main(int argc, char* argv[MUNIT_ARRAY_PARAM(argc + 1)]) { + /* Finally, we'll actually run our test suite! That second argument + * is the user_data parameter which will be passed either to the + * test or (if provided) the fixture setup function. */ + return munit_suite_main(&test_suite, (void*) "µnit", argc, argv); +} From 451c892b4deb261c7e74cfc9955ca8585e484191 Mon Sep 17 00:00:00 2001 From: Alvaro Garcia Date: Tue, 8 Feb 2022 13:09:08 +0400 Subject: [PATCH 3/3] [83] Execute all children test suites (covered by test) --- Makefile | 6 ++++++ demo.sh | 36 +++++++++++++++++++++++++++++++++ demo_expected.log | 51 +++++++++++++++++++++++++++++++++++++++++++++++ munit.c | 21 +++++++++++++------ 4 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 demo.sh create mode 100644 demo_expected.log diff --git a/Makefile b/Makefile index dc1ac54..cb2dd7c 100644 --- a/Makefile +++ b/Makefile @@ -56,3 +56,9 @@ clean: rm -f example$(EXTENSION) all: example$(EXTENSION) + +demo_actual.log: + rm -f demo_actual.log && touch demo_actual.log + bash ./demo.sh 2>&1 | tee -a demo_actual.log + diff -q demo_actual.log demo_expected.log +.PHONY: demo_actual.log diff --git a/demo.sh b/demo.sh new file mode 100644 index 0000000..ab0ba84 --- /dev/null +++ b/demo.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -euo pipefail + +function run_test(){ + local message="$1" + local suite=$2 + + # Only keep lines that are the executed tests + # Remove timing information (to keep the log reproducible) + echo "$message with pattern $suite" + set +e + # Do not quote suite, otherwise, it executes "" for the empty string + ./test_setup $suite | grep " OK " | cut -d "[" -f1 + set -e +} + +echo "# Happy Path Tests" + +run_test "Executing all" "" + +run_test "Executing at first level" "/perf" + +run_test "Executing at second level" "/perf/symmetric" + +run_test "Executing at third level" "/perf/symmetric/sha2" + +run_test "Executing at leaf level" "/perf/symmetric/sha2/rand" + +echo "# Not Happy Path Tests" + +run_test "Executing something that doesn't exist" "/xx" + +run_test "Executing something that doesn't exist (2)" "/xx/rand" + +echo "Script finished successfully" \ No newline at end of file diff --git a/demo_expected.log b/demo_expected.log new file mode 100644 index 0000000..dc28f45 --- /dev/null +++ b/demo_expected.log @@ -0,0 +1,51 @@ +# Happy Path Tests +Executing all with pattern +/kat/symmetric/sha2/compare +/kat/symmetric/sha2/rand +/kat/symmetric/sha3/compare +/kat/symmetric/sha3/rand +/kat/asymmetric/x25519/compare +/kat/asymmetric/x25519/rand +/kat/asymmetric/NIST/compare +/kat/asymmetric/NIST/rand +/unit/symmetric/sha2/compare +/unit/symmetric/sha2/rand +/unit/symmetric/sha3/compare +/unit/symmetric/sha3/rand +/unit/asymmetric/x25519/compare +/unit/asymmetric/x25519/rand +/unit/asymmetric/NIST/compare +/unit/asymmetric/NIST/rand +/perf/symmetric/sha2/compare +/perf/symmetric/sha2/rand +/perf/symmetric/sha3/compare +/perf/symmetric/sha3/rand +/perf/asymmetric/x25519/compare +/perf/asymmetric/x25519/rand +/perf/asymmetric/NIST/compare +/perf/asymmetric/NIST/rand +Executing at first level with pattern /perf +/perf/symmetric/sha2/compare +/perf/symmetric/sha2/rand +/perf/symmetric/sha3/compare +/perf/symmetric/sha3/rand +/perf/asymmetric/x25519/compare +/perf/asymmetric/x25519/rand +/perf/asymmetric/NIST/compare +/perf/asymmetric/NIST/rand +Executing at second level with pattern /perf/symmetric +/perf/symmetric/sha2/compare +/perf/symmetric/sha2/rand +/perf/symmetric/sha3/compare +/perf/symmetric/sha3/rand +Executing at third level with pattern /perf/symmetric/sha2 +/perf/symmetric/sha2/compare +/perf/symmetric/sha2/rand +Executing at leaf level with pattern /perf/symmetric/sha2/rand +/perf/symmetric/sha2/rand +# Not Happy Path Tests +Executing something that doesn't exist with pattern /xx +No tests run, 0 (100%) skipped. +Executing something that doesn't exist (2) with pattern /xx/rand +No tests run, 0 (100%) skipped. +Script finished successfully diff --git a/munit.c b/munit.c index 00ede07..f80f328 100644 --- a/munit.c +++ b/munit.c @@ -1655,16 +1655,27 @@ munit_test_runner_run_test(MunitTestRunner* runner, static void munit_test_runner_run_suite(MunitTestRunner* runner, const MunitSuite* suite, - const char* prefix) { + const char* prefix, + bool forced) { size_t pre_l; char* pre = munit_maybe_concat(&pre_l, (char*) prefix, (char*) suite->prefix); const MunitTest* test; const char** test_name; const MunitSuite* child_suite; + bool is_top = pre_l == 0; + const char **user_submitted_ptr = runner->tests; + bool run_all_tests = user_submitted_ptr == NULL; + forced |= (true + && suite->prefix != NULL + && strlen(suite->prefix) > 0 + && user_submitted_ptr != NULL + && strcmp(*user_submitted_ptr, pre) == 0); /* Run the tests. */ for (test = suite->tests ; test != NULL && test->test != NULL ; test++) { - if (runner->tests != NULL) { /* Specific tests were requested on the CLI */ + if (forced || run_all_tests) { + munit_test_runner_run_test(runner, test, pre); + } else { /* Specific tests were requested on the CLI */ for (test_name = runner->tests ; test_name != NULL && *test_name != NULL ; test_name++) { if ((pre_l == 0 || strncmp(pre, *test_name, pre_l) == 0) && strncmp(test->name, *test_name + pre_l, strlen(*test_name + pre_l)) == 0) { @@ -1673,8 +1684,6 @@ munit_test_runner_run_suite(MunitTestRunner* runner, goto cleanup; } } - } else { /* Run all tests */ - munit_test_runner_run_test(runner, test, pre); } } @@ -1683,7 +1692,7 @@ munit_test_runner_run_suite(MunitTestRunner* runner, /* Run any child suites. */ for (child_suite = suite->suites ; child_suite != NULL && child_suite->prefix != NULL ; child_suite++) { - munit_test_runner_run_suite(runner, child_suite, pre); + munit_test_runner_run_suite(runner, child_suite, pre, forced); } cleanup: @@ -1693,7 +1702,7 @@ munit_test_runner_run_suite(MunitTestRunner* runner, static void munit_test_runner_run(MunitTestRunner* runner) { - munit_test_runner_run_suite(runner, runner->suite, NULL); + munit_test_runner_run_suite(runner, runner->suite, NULL, false); } static void