Reinstate old test suite (#645)

* Reinstate old test suite

This is a simple output comparison based testsuite.

The previous version of this is on the `old-version` branch.

* Windows line endings fix.

.github/workflows/cmake.yml

@@ -59,7 +59,7 @@ jobs:
 
     - name: Build
       # Build your program with the given configuration
-      run: cmake --build ${{github.workspace}}/build --config ${{matrix.build-type}}
+      run: cmake --build ${{github.workspace}}/build --config ${{matrix.build-type}} --target install
 
     - name: Test
       working-directory: ${{github.workspace}}/build

CMakeLists.txt

@@ -44,3 +44,4 @@ install(DIRECTORY std DESTINATION verona)
 
 enable_testing()
 add_subdirectory(src)
+add_subdirectory(testsuite)

src/parse.cc

@@ -75,21 +75,21 @@ namespace verona
     p("start",
       {
         // Blank lines terminate.
-        "\n(?:[[:blank:]]*\n)+([[:blank:]]*)" >>
+        "\r?\n(?:[[:blank:]]*\r?\n)+([[:blank:]]*)" >>
           [indent](auto& m) {
             indent->back() = m.match(1).len;
             m.term(terminators);
           },
 
         // A newline that starts a brace block doesn't terminate.
-        "\n([[:blank:]]*(\\{)[[:blank:]]*)" >>
+        "\r?\n([[:blank:]]*(\\{)[[:blank:]]*)" >>
           [indent](auto& m) {
             indent->push_back(m.match(1).len);
             m.push(Brace, 2);
           },
 
         // A newline sometimes terminates.
-        "\n([[:blank:]]*)" >>
+        "\r?\n([[:blank:]]*)" >>
           [indent](auto& m) {
             size_t col = m.match(1).len;
 
@@ -216,7 +216,7 @@ namespace verona
         ":\\[((?:[^\\]]|\\][^:])*)\\]:" >> [](auto& m) { m.add(LLVM, 1); },
 
         // Line comment.
-        "//[^\n]*" >> [](auto&) {},
+        "//[^\r\n]*" >> [](auto&) {},
 
         // Nested comment.
         "/\\*" >>

testsuite/CMakeLists.txt

@@ -0,0 +1,150 @@
+enable_testing()
+
+find_program(DIFF_TOOL NAMES
+  diff)
+
+if (DIFF_TOOL STREQUAL DIFF_TOOL-NOTFOUND)
+  set(DIFF_TOOL "")
+endif()
+
+set(DISABLED_TESTS)
+
+set(GOLDEN_DIR_SUFFIX "_out")
+
+set(VERONA_LOCAL_DIST ${CMAKE_BINARY_DIR}/dist)
+message("Verona local dist: ${VERONA_LOCAL_DIST}")
+
+function(subdirlist result curdir)
+  file(GLOB children LIST_DIRECTORIES true CONFIGURE_DEPENDS RELATIVE ${curdir}  ${curdir}/*)
+  set(dirlist "")
+  foreach(child ${children})
+    if(IS_DIRECTORY ${curdir}/${child})
+      list(APPEND dirlist ${child})
+    endif()
+  endforeach()
+  set(${result} ${dirlist} PARENT_SCOPE)
+endfunction()
+
+# Iterate each tool
+subdirlist(TOOL_FOLDERS ${CMAKE_CURRENT_SOURCE_DIR})
+set(UPDATE_DUMPS_TARGETS)
+foreach(TOOL ${TOOL_FOLDERS})
+  if (TOOL MATCHES "deprecated")
+    continue()
+  endif()
+  set (test_set)
+  set(TOOL_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}/${TOOL})
+
+  # Grab specific settings for this tool
+  include(${CMAKE_CURRENT_LIST_DIR}/${TOOL}.cmake)
+
+  # Use transform to support multiple extension for tests,
+  # and find all the files with these extensions.
+  list(TRANSFORM TEST_EXTENSION PREPEND ${TOOL_FOLDER}/*.)
+  file(GLOB_RECURSE tests CONFIGURE_DEPENDS  RELATIVE ${TOOL_FOLDER} ${TEST_EXTENSION})
+
+  foreach(test ${tests})
+    get_filename_component(test_name ${test} NAME_WE)
+    get_filename_component(test_file ${test} NAME)
+    get_filename_component(test_dir ${test} DIRECTORY)
+
+    if (test_dir STREQUAL "")
+      set (test_path ${TOOL}/${test_name})
+    else()
+      set (test_path ${TOOL}/${test_dir}/${test_name})
+    endif()
+
+    list (FIND DISABLED_TESTS ${test_path} INDEX)
+    if (NOT ${INDEX} EQUAL -1)
+      message("Test currently disabled ${test_path}")
+      continue()
+    endif()
+
+    # Create command to create the output for this test.
+    set (output_dir ${CMAKE_CURRENT_BINARY_DIR}/${test_path}${GOLDEN_DIR_SUFFIX})
+    set (test_output_cmd 
+      ${CMAKE_COMMAND}
+        -DTESTFILE=${test_file}
+        -DWORKING_DIR=${TOOL_FOLDER}/${test_dir}
+        -DTOOLNAME=${TOOL}
+        -DCMAKE_EXECUTABLE_SUFFIX=${CMAKE_EXECUTABLE_SUFFIX}
+        -DVERONA_LOCAL_DIST=${VERONA_LOCAL_DIST}
+        -DOUTPUT_DIR=${output_dir}
+        -P ${CMAKE_CURRENT_SOURCE_DIR}/run_command.cmake
+    )
+
+    # Add test that rebuilds the compiler output
+    add_test(NAME ${test_path}${GOLDEN_DIR_SUFFIX}
+      COMMAND ${test_output_cmd}
+    )
+
+    # Add command that rebuilts the compiler output for updating golden files.
+    add_custom_command(OUTPUT ${test_path}
+      COMMAND ${test_output_cmd}
+    )
+    set_source_files_properties(${test_path} PROPERTIES SYMBOLIC "true")
+    list(APPEND test_set ${test_path})
+
+    # Add output comparison for each golden / output file
+    set (golden_dir  ${CMAKE_CURRENT_SOURCE_DIR}/${test_path}${GOLDEN_DIR_SUFFIX} )
+    file (GLOB_RECURSE results CONFIGURE_DEPENDS RELATIVE ${golden_dir} ${golden_dir}/*)
+    # Check if there are any files to compare for this test.
+    list(LENGTH results res_length)
+    if(res_length EQUAL 0)
+      message(WARNING "Test does not have results directory: ${golden_dir}")
+      # Add to generate golden output target
+      add_custom_command(OUTPUT ${test_path}
+        COMMAND
+          ${CMAKE_COMMAND}
+          -E make_directory
+          ${golden_dir}
+        APPEND
+      )
+      add_custom_command(OUTPUT ${test_path}
+        COMMAND
+          ${CMAKE_COMMAND}
+          -E copy_if_different
+          ${output_dir}/*
+          ${golden_dir}/
+        APPEND
+      )
+    else()
+      foreach (result ${results})
+        # Check each file is correct as a test target
+        add_test (NAME ${test_path}${GOLDEN_DIR_SUFFIX}/${result}
+          COMMAND 
+            ${CMAKE_COMMAND}
+              -Doriginal_file=${golden_dir}/${result} 
+              -Dnew_file=${output_dir}/${result}
+              -Ddiff_tool=${DIFF_TOOL}
+              -P ${CMAKE_CURRENT_SOURCE_DIR}/compare.cmake
+        )
+        set_tests_properties(${test_path}${GOLDEN_DIR_SUFFIX}/${result} PROPERTIES DEPENDS ${test_path}${GOLDEN_DIR_SUFFIX})
+
+        # Override out of date files.
+        add_custom_command(OUTPUT ${test_path}
+          COMMAND
+            ${CMAKE_COMMAND}
+            -E copy_if_different
+            ${output_dir}/${result}
+            ${golden_dir}/${result}
+          APPEND
+        )
+      endforeach()
+      # All tests require an error_code.
+      add_custom_command(OUTPUT ${test_path}
+        COMMAND
+          ${CMAKE_COMMAND}
+          -E copy_if_different
+          ${output_dir}/exit_code.txt
+          ${golden_dir}/exit_code.txt
+        APPEND
+      )
+
+    endif()
+  endforeach()
+  add_custom_target("update-dump-${TOOL}" DEPENDS ${test_set})
+  list(APPEND UPDATE_DUMPS_TARGETS "update-dump-${TOOL}")
+endforeach()
+
+add_custom_target(update-dump DEPENDS ${UPDATE_DUMPS_TARGETS})

testsuite/README.md

@@ -0,0 +1,48 @@
+# Verona testsuite
+
+Test cases are divided in top-level directories, for each tool they are testing.
+For each top-level directory there is a [toplevel].cmake, e.g. `verona.cmake` is the
+settings for the `verona` directory, which is used to test `verona`.   The `cmake` file
+provides two things:
+
+* A macro to provide the invoke the tool with appropriate command line arguments given the
+  test file, and the output directory.
+* A list of extensions for tests, e.g.  `set(TEST_EXTENSION verona)`. 
+
+For each test, there is an associate golden output directory. For example, the test
+```
+testsuite/verona/basic/ex1.verona
+```
+has an associated output directory
+```
+testsuite/verona/basic/ex1.out/
+```
+This can contain any files that were generated by the command using the output directory. 
+Additionally, the harness can generate `stdout.txt`, `stderr.txt` and `exit_code.txt` for the 
+values of the output and error streams, and the exit code of the test.  If the directory does
+not exist CMake will emit a warning.  Only files in the directory will be checked that the tool
+generated the correct files.
+
+## Updating tests
+
+The files that exist in the `.out` directories can be automatically updated.  There is a build
+target `update-dump` that will automatically update the contents of any `.out` directory.  It
+will only update the files that already exist in the directory, so if there is no `stdout.txt`
+file, then after calling `update-dump` the file will still not exist.  If you wish to add the file,
+create the file, and then run `update-dump`.
+
+There are also targets for individual testsuite directories, to just regenerate the tests in those
+directories. For instance, `verona-update-dump` will just update the output in that directory.
+
+##  Adding new tests
+
+The `update-dump` target will also generate new output directories for any test that does not
+currently have an output directory.  The output directory will contain all the output files that
+the test can generate.  You might not wish to test all the phases with this test. Remove any files
+you do not wish to test, and commit the rest.
+
+## To add a new tool
+
+If you need to add a new tool, or a new invocation of a tool, then you create a new top-level 
+directory, and create the `[directoryname].cmake` file to find tests and generate
+the command line.

testsuite/compare.cmake

@@ -0,0 +1,32 @@
+# This is used to compare to files using CMake.
+# It improves on the default behaviour to print the
+# the files if there is a difference.
+
+execute_process(
+    COMMAND ${CMAKE_COMMAND} -E compare_files --ignore-eol ${original_file} ${new_file}
+    RESULT_VARIABLE status
+)
+
+if (${status} EQUAL 1)
+    message ("Compare ${original_file} with ${new_file}")
+    if (diff_tool STREQUAL "")
+        file(READ ${original_file} original_text)
+        file(READ ${new_file} new_text)
+        message("--Original File-----------------------------------------------------------------")
+        if (NOT original_text STREQUAL "")
+            message("${original_text}")
+        endif()
+        message("--------------------------------------------------------------------------------")
+        message("  ")
+        message("--New File----------------------------------------------------------------------")
+        if (NOT new_text STREQUAL "")
+            message(${new_text})
+        endif()
+        message("--------------------------------------------------------------------------------")
+    else ()
+        execute_process(
+            COMMAND ${diff_tool} ${original_file} ${new_file}
+        )
+    endif ()
+    message(FATAL_ERROR "Files differ!")
+endif ()

testsuite/run_command.cmake

@@ -0,0 +1,35 @@
+# This is used to run a command that can fail.
+# Dumping all the output and error code into a file
+# Also handles timeouts
+
+file(REMOVE_RECURSE ${OUTPUT_DIR})
+
+make_directory(${OUTPUT_DIR})
+
+include(${CMAKE_CURRENT_LIST_DIR}/${TOOLNAME}.cmake)
+
+message("Verona Local dist: ${VERONA_LOCAL_DIST}")
+toolinvoke(TOOLINVOKE ${VERONA_LOCAL_DIST} ${TESTFILE} ${OUTPUT_DIR})
+
+list(JOIN TOOLINVOKE " " TOOLINVOKE_SEP)
+message ("Running")
+message ("   ${TOOLINVOKE_SEP}")
+message ("in working directory")
+message ("   ${WORKING_DIR}")
+message ("output sent to")
+message ("   ${OUTPUT_DIR}")
+
+# Run command
+execute_process(
+    COMMAND ${TOOLINVOKE}
+    WORKING_DIRECTORY ${WORKING_DIR}
+    OUTPUT_FILE ${OUTPUT_DIR}/stdout.txt
+    ERROR_FILE ${OUTPUT_DIR}/stderr.txt
+    TIMEOUT 20   # Timeout at 20 seconds, may need to increase this.
+    RESULT_VARIABLE status
+)
+
+# Push exit code into dump and make sure both stdout and stderr exist
+file(WRITE ${OUTPUT_DIR}/exit_code.txt ${status})
+file(TOUCH ${OUTPUT_DIR}/stdout.txt)
+file(TOUCH ${OUTPUT_DIR}/stderr.txt)

testsuite/verona.cmake

@@ -0,0 +1,7 @@
+# Arguments for testing verona-parser
+macro(toolinvoke ARGS local_dist testfile outputdir)
+  set(${ARGS} "${local_dist}/verona/verona${CMAKE_EXECUTABLE_SUFFIX}" build ${testfile} --wf-check -o ${outputdir}/ast)
+endmacro()
+
+# Set extension for verona tests
+set(TEST_EXTENSION "verona")

testsuite/verona/basic/ex1.verona

@@ -0,0 +1,5 @@
+class None {
+  create() : None {
+    new ()
+  }
+}

testsuite/verona/basic/ex1_out/exit_code.txt

@@ -0,0 +1 @@
+0