From b69874736d082557feb966a2d3dd96e51e8ed90d Mon Sep 17 00:00:00 2001
From: Neil Henderson <neil@bluetarpmedia.com>
Date: Sat, 2 Nov 2024 10:36:09 +1000
Subject: [PATCH] [CI] run-tests.sh now allows each compiler config to exclude
 test files that it doesn't want to run

There are 2 macOS test configs which share the same expected results directory (to avoid duplication). This works great except for the new `pure2-expected-is-as` test. This new test code fails to compile (as expected) on both compilers, but produces a slightly different error diagnostic because the path is different.

E.g.

/Applications/Xcode_14.3.1.app/...etc.../math.h
and
/Library/Developer/CommandLineTools/...etc.../math.h

One option would be to stop sharing the expected results for both of these compilers, but that seems wasteful since it's just one test which fails to compile. So instead the `run-tests` script has a new way to exclude a test from running.
---
 regression-tests/run-tests.sh | 53 ++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 20 deletions(-)

diff --git a/regression-tests/run-tests.sh b/regression-tests/run-tests.sh
index b4cfc8787..6e3ac81b7 100644
--- a/regression-tests/run-tests.sh
+++ b/regression-tests/run-tests.sh
@@ -135,28 +135,11 @@ if [ -z "$label" ]; then
     usage
 fi
 
-tests=$(ls | grep ".cpp2$")
-if [[ -n "$chosen_tests" ]]; then
-    for test in $chosen_tests; do
-        if ! [[ -f "$test" ]]; then
-            echo "Requested test ($test) not found"
-            exit 1
-        fi
-    done
-    echo "Performing tests:"
-    for test in $chosen_tests; do
-        echo "    $test"
-    done
-    echo
-    tests="$chosen_tests"
-else
-    printf "Performing all regression tests\n\n"
-fi
-
-expected_results_dir="test-results"
-
 ################
 # Get the directory with the exec outputs and compilation command
+# We also allow each compiler configuration to specify any test files(s) to exclude from running.
+expected_results_dir="test-results"
+exclude_test_filter=""
 if [[ "$cxx_compiler" == *"cl.exe"* ]]; then
     compiler_cmd="cl.exe -nologo -std:${cxx_std} -MD -EHsc -I ..\..\..\include -Fe:"
     exec_out_dir="$expected_results_dir/msvc-2022-${cxx_std}"
@@ -174,6 +157,12 @@ else
     if [[ "$compiler_version" == *"Apple clang version 14.0"* ||
           "$compiler_version" == *"Homebrew clang version 15.0"* ]]; then
         exec_out_dir="$expected_results_dir/apple-clang-14"
+        # We share the expected results dir for these two compilers, but there is one
+        # test which (as expected) fails to compile on both compilers, but has a slightly
+        # different error diagnostic because the clang path differs. So we exclude it from
+        # running. The alternative would be to duplicate the expected results files, which
+        # seems wasteful for just one test (that doesn't even compile).
+        exclude_test_filter="pure2-expected-is-as.cpp2"
     elif [[ "$compiler_version" == *"Apple clang version 15.0"* ]]; then
         exec_out_dir="$expected_results_dir/apple-clang-15"
     elif [[ "$compiler_version" == *"clang version 12.0"* ]]; then 
@@ -236,6 +225,30 @@ else
     exit 2
 fi
 
+################
+# Get the list of .cpp2 test files
+if [[ -n "$exclude_test_filter" ]]; then
+    tests=$(ls | grep ".cpp2$" | grep -v $exclude_test_filter)
+else
+    tests=$(ls | grep ".cpp2$")
+fi
+if [[ -n "$chosen_tests" ]]; then
+    for test in $chosen_tests; do
+        if ! [[ -f "$test" ]]; then
+            echo "Requested test ($test) not found"
+            exit 1
+        fi
+    done
+    echo "Performing tests:"
+    for test in $chosen_tests; do
+        echo "    $test"
+    done
+    echo
+    tests="$chosen_tests"
+else
+    printf "Performing all regression tests\n\n"
+fi
+
 ################
 cppfront_cmd="cppfront.exe"
 echo "Building cppfront"