Feature/83 iterate all children test suites

.gitignore

@@ -4,3 +4,9 @@
 /example
 /*.exe
 /*.dll
+
+.idea/
+cmake-*/
+
+*_actual.log
+

Makefile

@@ -46,10 +46,19 @@ endif
 example$(EXTENSION): munit.h munit.c example.c
 	$(CC) $(CFLAGS) -o $@ munit.c example.c
 
+test_setup$(EXTENSION): munit.h munit.c test_setup.c
+	$(CC) $(CFLAGS) -o $@ munit.c test_setup.c
+
 test:
 	$(TEST_ENV) ./example$(EXTENSION)
 
 clean:
 	rm -f example$(EXTENSION)
 
 all: example$(EXTENSION)
+
+demo_actual.log:
+	rm -f demo_actual.log && touch demo_actual.log
+	bash ./demo.sh 2>&1 | tee -a demo_actual.log
+	diff -q demo_actual.log demo_expected.log
+.PHONY: demo_actual.log

demo.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+function run_test(){
+  local message="$1"
+  local suite=$2
+
+  # Only keep lines that are the executed tests
+  # Remove timing information (to keep the log reproducible)
+  echo "$message with pattern $suite"
+  set +e
+  # Do not quote suite, otherwise, it executes "" for the empty string
+  ./test_setup $suite | grep " OK " | cut -d "[" -f1
+  set -e
+}
+
+echo "# Happy Path Tests"
+
+run_test "Executing all" ""
+
+run_test "Executing at first level" "/perf"
+
+run_test "Executing at second level" "/perf/symmetric"
+
+run_test "Executing at third level" "/perf/symmetric/sha2"
+
+run_test "Executing at leaf level" "/perf/symmetric/sha2/rand"
+
+echo "# Not Happy Path Tests"
+
+run_test "Executing something that doesn't exist" "/xx"
+
+run_test "Executing something that doesn't exist (2)" "/xx/rand"
+
+echo "Script finished successfully"

demo_expected.log

@@ -0,0 +1,51 @@
+# Happy Path Tests
+Executing all with pattern 
+/kat/symmetric/sha2/compare          
+/kat/symmetric/sha2/rand             
+/kat/symmetric/sha3/compare          
+/kat/symmetric/sha3/rand             
+/kat/asymmetric/x25519/compare       
+/kat/asymmetric/x25519/rand          
+/kat/asymmetric/NIST/compare         
+/kat/asymmetric/NIST/rand            
+/unit/symmetric/sha2/compare         
+/unit/symmetric/sha2/rand            
+/unit/symmetric/sha3/compare         
+/unit/symmetric/sha3/rand            
+/unit/asymmetric/x25519/compare      
+/unit/asymmetric/x25519/rand         
+/unit/asymmetric/NIST/compare        
+/unit/asymmetric/NIST/rand           
+/perf/symmetric/sha2/compare         
+/perf/symmetric/sha2/rand            
+/perf/symmetric/sha3/compare         
+/perf/symmetric/sha3/rand            
+/perf/asymmetric/x25519/compare      
+/perf/asymmetric/x25519/rand         
+/perf/asymmetric/NIST/compare        
+/perf/asymmetric/NIST/rand           
+Executing at first level with pattern /perf
+/perf/symmetric/sha2/compare         
+/perf/symmetric/sha2/rand            
+/perf/symmetric/sha3/compare         
+/perf/symmetric/sha3/rand            
+/perf/asymmetric/x25519/compare      
+/perf/asymmetric/x25519/rand         
+/perf/asymmetric/NIST/compare        
+/perf/asymmetric/NIST/rand           
+Executing at second level with pattern /perf/symmetric
+/perf/symmetric/sha2/compare         
+/perf/symmetric/sha2/rand            
+/perf/symmetric/sha3/compare         
+/perf/symmetric/sha3/rand            
+Executing at third level with pattern /perf/symmetric/sha2
+/perf/symmetric/sha2/compare         
+/perf/symmetric/sha2/rand            
+Executing at leaf level with pattern /perf/symmetric/sha2/rand
+/perf/symmetric/sha2/rand            
+# Not Happy Path Tests
+Executing something that doesn't exist with pattern /xx
+No tests run, 0 (100%) skipped.
+Executing something that doesn't exist (2) with pattern /xx/rand
+No tests run, 0 (100%) skipped.
+Script finished successfully

munit.c

@@ -1655,16 +1655,27 @@ munit_test_runner_run_test(MunitTestRunner* runner,
 static void
 munit_test_runner_run_suite(MunitTestRunner* runner,
                             const MunitSuite* suite,
-                            const char* prefix) {
+                            const char* prefix,
+                            bool forced) {
   size_t pre_l;
   char* pre = munit_maybe_concat(&pre_l, (char*) prefix, (char*) suite->prefix);
   const MunitTest* test;
   const char** test_name;
   const MunitSuite* child_suite;
+  bool is_top = pre_l == 0;
+  const char **user_submitted_ptr = runner->tests;
+  bool run_all_tests = user_submitted_ptr == NULL;
+  forced |= (true
+             && suite->prefix != NULL
+             && strlen(suite->prefix) > 0
+             && user_submitted_ptr != NULL
+             && strcmp(*user_submitted_ptr, pre) == 0);
 
   /* Run the tests. */
   for (test = suite->tests ; test != NULL && test->test != NULL ; test++) {
-    if (runner->tests != NULL) { /* Specific tests were requested on the CLI */
+    if (forced || run_all_tests) {
+      munit_test_runner_run_test(runner, test, pre);
+    } else { /* Specific tests were requested on the CLI */
       for (test_name = runner->tests ; test_name != NULL && *test_name != NULL ; test_name++) {
         if ((pre_l == 0 || strncmp(pre, *test_name, pre_l) == 0) &&
             strncmp(test->name, *test_name + pre_l, strlen(*test_name + pre_l)) == 0) {
@@ -1673,8 +1684,6 @@ munit_test_runner_run_suite(MunitTestRunner* runner,
             goto cleanup;
         }
       }
-    } else { /* Run all tests */
-      munit_test_runner_run_test(runner, test, pre);
     }
   }
 
@@ -1683,7 +1692,7 @@ munit_test_runner_run_suite(MunitTestRunner* runner,
 
   /* Run any child suites. */
   for (child_suite = suite->suites ; child_suite != NULL && child_suite->prefix != NULL ; child_suite++) {
-    munit_test_runner_run_suite(runner, child_suite, pre);
+    munit_test_runner_run_suite(runner, child_suite, pre, forced);
   }
 
  cleanup:
@@ -1693,7 +1702,7 @@ munit_test_runner_run_suite(MunitTestRunner* runner,
 
 static void
 munit_test_runner_run(MunitTestRunner* runner) {
-  munit_test_runner_run_suite(runner, runner->suite, NULL);
+  munit_test_runner_run_suite(runner, runner->suite, NULL, false);
 }
 
 static void