From 24aef09a54055852a82ea0401218469a7244fd2d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 5 Oct 2023 08:56:55 +0100 Subject: [PATCH] Reinstate old test suite (#645) * Reinstate old test suite This is a simple output comparison based testsuite. The previous version of this is on the `old-version` branch. * Windows line endings fix. --- .github/workflows/cmake.yml | 2 +- CMakeLists.txt | 1 + src/parse.cc | 8 +- testsuite/CMakeLists.txt | 150 +++++++++++++++++++ testsuite/README.md | 48 ++++++ testsuite/compare.cmake | 32 ++++ testsuite/run_command.cmake | 35 +++++ testsuite/verona.cmake | 7 + testsuite/verona/basic/ex1.verona | 5 + testsuite/verona/basic/ex1_out/exit_code.txt | 1 + testsuite/verona/basic/ex1_out/stderr.txt | 0 testsuite/verona/basic/ex1_out/stdout.txt | 0 12 files changed, 284 insertions(+), 5 deletions(-) create mode 100644 testsuite/CMakeLists.txt create mode 100644 testsuite/README.md create mode 100644 testsuite/compare.cmake create mode 100644 testsuite/run_command.cmake create mode 100644 testsuite/verona.cmake create mode 100755 testsuite/verona/basic/ex1.verona create mode 100644 testsuite/verona/basic/ex1_out/exit_code.txt create mode 100644 testsuite/verona/basic/ex1_out/stderr.txt create mode 100644 testsuite/verona/basic/ex1_out/stdout.txt diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 50a7cf74d..3a575752e 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -59,7 +59,7 @@ jobs: - name: Build # Build your program with the given configuration - run: cmake --build ${{github.workspace}}/build --config ${{matrix.build-type}} + run: cmake --build ${{github.workspace}}/build --config ${{matrix.build-type}} --target install - name: Test working-directory: ${{github.workspace}}/build diff --git a/CMakeLists.txt b/CMakeLists.txt index dc82b31fa..b7f1147ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,3 +44,4 @@ install(DIRECTORY std DESTINATION verona) enable_testing() add_subdirectory(src) +add_subdirectory(testsuite) \ No newline at end of file diff --git a/src/parse.cc b/src/parse.cc index 445e2fac8..d0a0e32c1 100644 --- a/src/parse.cc +++ b/src/parse.cc @@ -75,21 +75,21 @@ namespace verona p("start", { // Blank lines terminate. - "\n(?:[[:blank:]]*\n)+([[:blank:]]*)" >> + "\r?\n(?:[[:blank:]]*\r?\n)+([[:blank:]]*)" >> [indent](auto& m) { indent->back() = m.match(1).len; m.term(terminators); }, // A newline that starts a brace block doesn't terminate. - "\n([[:blank:]]*(\\{)[[:blank:]]*)" >> + "\r?\n([[:blank:]]*(\\{)[[:blank:]]*)" >> [indent](auto& m) { indent->push_back(m.match(1).len); m.push(Brace, 2); }, // A newline sometimes terminates. - "\n([[:blank:]]*)" >> + "\r?\n([[:blank:]]*)" >> [indent](auto& m) { size_t col = m.match(1).len; @@ -216,7 +216,7 @@ namespace verona ":\\[((?:[^\\]]|\\][^:])*)\\]:" >> [](auto& m) { m.add(LLVM, 1); }, // Line comment. - "//[^\n]*" >> [](auto&) {}, + "//[^\r\n]*" >> [](auto&) {}, // Nested comment. "/\\*" >> diff --git a/testsuite/CMakeLists.txt b/testsuite/CMakeLists.txt new file mode 100644 index 000000000..12489d6a3 --- /dev/null +++ b/testsuite/CMakeLists.txt @@ -0,0 +1,150 @@ +enable_testing() + +find_program(DIFF_TOOL NAMES + diff) + +if (DIFF_TOOL STREQUAL DIFF_TOOL-NOTFOUND) + set(DIFF_TOOL "") +endif() + +set(DISABLED_TESTS) + +set(GOLDEN_DIR_SUFFIX "_out") + +set(VERONA_LOCAL_DIST ${CMAKE_BINARY_DIR}/dist) +message("Verona local dist: ${VERONA_LOCAL_DIST}") + +function(subdirlist result curdir) + file(GLOB children LIST_DIRECTORIES true CONFIGURE_DEPENDS RELATIVE ${curdir} ${curdir}/*) + set(dirlist "") + foreach(child ${children}) + if(IS_DIRECTORY ${curdir}/${child}) + list(APPEND dirlist ${child}) + endif() + endforeach() + set(${result} ${dirlist} PARENT_SCOPE) +endfunction() + +# Iterate each tool +subdirlist(TOOL_FOLDERS ${CMAKE_CURRENT_SOURCE_DIR}) +set(UPDATE_DUMPS_TARGETS) +foreach(TOOL ${TOOL_FOLDERS}) + if (TOOL MATCHES "deprecated") + continue() + endif() + set (test_set) + set(TOOL_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}/${TOOL}) + + # Grab specific settings for this tool + include(${CMAKE_CURRENT_LIST_DIR}/${TOOL}.cmake) + + # Use transform to support multiple extension for tests, + # and find all the files with these extensions. + list(TRANSFORM TEST_EXTENSION PREPEND ${TOOL_FOLDER}/*.) + file(GLOB_RECURSE tests CONFIGURE_DEPENDS RELATIVE ${TOOL_FOLDER} ${TEST_EXTENSION}) + + foreach(test ${tests}) + get_filename_component(test_name ${test} NAME_WE) + get_filename_component(test_file ${test} NAME) + get_filename_component(test_dir ${test} DIRECTORY) + + if (test_dir STREQUAL "") + set (test_path ${TOOL}/${test_name}) + else() + set (test_path ${TOOL}/${test_dir}/${test_name}) + endif() + + list (FIND DISABLED_TESTS ${test_path} INDEX) + if (NOT ${INDEX} EQUAL -1) + message("Test currently disabled ${test_path}") + continue() + endif() + + # Create command to create the output for this test. + set (output_dir ${CMAKE_CURRENT_BINARY_DIR}/${test_path}${GOLDEN_DIR_SUFFIX}) + set (test_output_cmd + ${CMAKE_COMMAND} + -DTESTFILE=${test_file} + -DWORKING_DIR=${TOOL_FOLDER}/${test_dir} + -DTOOLNAME=${TOOL} + -DCMAKE_EXECUTABLE_SUFFIX=${CMAKE_EXECUTABLE_SUFFIX} + -DVERONA_LOCAL_DIST=${VERONA_LOCAL_DIST} + -DOUTPUT_DIR=${output_dir} + -P ${CMAKE_CURRENT_SOURCE_DIR}/run_command.cmake + ) + + # Add test that rebuilds the compiler output + add_test(NAME ${test_path}${GOLDEN_DIR_SUFFIX} + COMMAND ${test_output_cmd} + ) + + # Add command that rebuilts the compiler output for updating golden files. + add_custom_command(OUTPUT ${test_path} + COMMAND ${test_output_cmd} + ) + set_source_files_properties(${test_path} PROPERTIES SYMBOLIC "true") + list(APPEND test_set ${test_path}) + + # Add output comparison for each golden / output file + set (golden_dir ${CMAKE_CURRENT_SOURCE_DIR}/${test_path}${GOLDEN_DIR_SUFFIX} ) + file (GLOB_RECURSE results CONFIGURE_DEPENDS RELATIVE ${golden_dir} ${golden_dir}/*) + # Check if there are any files to compare for this test. + list(LENGTH results res_length) + if(res_length EQUAL 0) + message(WARNING "Test does not have results directory: ${golden_dir}") + # Add to generate golden output target + add_custom_command(OUTPUT ${test_path} + COMMAND + ${CMAKE_COMMAND} + -E make_directory + ${golden_dir} + APPEND + ) + add_custom_command(OUTPUT ${test_path} + COMMAND + ${CMAKE_COMMAND} + -E copy_if_different + ${output_dir}/* + ${golden_dir}/ + APPEND + ) + else() + foreach (result ${results}) + # Check each file is correct as a test target + add_test (NAME ${test_path}${GOLDEN_DIR_SUFFIX}/${result} + COMMAND + ${CMAKE_COMMAND} + -Doriginal_file=${golden_dir}/${result} + -Dnew_file=${output_dir}/${result} + -Ddiff_tool=${DIFF_TOOL} + -P ${CMAKE_CURRENT_SOURCE_DIR}/compare.cmake + ) + set_tests_properties(${test_path}${GOLDEN_DIR_SUFFIX}/${result} PROPERTIES DEPENDS ${test_path}${GOLDEN_DIR_SUFFIX}) + + # Override out of date files. + add_custom_command(OUTPUT ${test_path} + COMMAND + ${CMAKE_COMMAND} + -E copy_if_different + ${output_dir}/${result} + ${golden_dir}/${result} + APPEND + ) + endforeach() + # All tests require an error_code. + add_custom_command(OUTPUT ${test_path} + COMMAND + ${CMAKE_COMMAND} + -E copy_if_different + ${output_dir}/exit_code.txt + ${golden_dir}/exit_code.txt + APPEND + ) + + endif() + endforeach() + add_custom_target("update-dump-${TOOL}" DEPENDS ${test_set}) + list(APPEND UPDATE_DUMPS_TARGETS "update-dump-${TOOL}") +endforeach() + +add_custom_target(update-dump DEPENDS ${UPDATE_DUMPS_TARGETS}) diff --git a/testsuite/README.md b/testsuite/README.md new file mode 100644 index 000000000..b2f196f92 --- /dev/null +++ b/testsuite/README.md @@ -0,0 +1,48 @@ +# Verona testsuite + +Test cases are divided in top-level directories, for each tool they are testing. +For each top-level directory there is a [toplevel].cmake, e.g. `verona.cmake` is the +settings for the `verona` directory, which is used to test `verona`. The `cmake` file +provides two things: + +* A macro to provide the invoke the tool with appropriate command line arguments given the + test file, and the output directory. +* A list of extensions for tests, e.g. `set(TEST_EXTENSION verona)`. + +For each test, there is an associate golden output directory. For example, the test +``` +testsuite/verona/basic/ex1.verona +``` +has an associated output directory +``` +testsuite/verona/basic/ex1.out/ +``` +This can contain any files that were generated by the command using the output directory. +Additionally, the harness can generate `stdout.txt`, `stderr.txt` and `exit_code.txt` for the +values of the output and error streams, and the exit code of the test. If the directory does +not exist CMake will emit a warning. Only files in the directory will be checked that the tool +generated the correct files. + +## Updating tests + +The files that exist in the `.out` directories can be automatically updated. There is a build +target `update-dump` that will automatically update the contents of any `.out` directory. It +will only update the files that already exist in the directory, so if there is no `stdout.txt` +file, then after calling `update-dump` the file will still not exist. If you wish to add the file, +create the file, and then run `update-dump`. + +There are also targets for individual testsuite directories, to just regenerate the tests in those +directories. For instance, `verona-update-dump` will just update the output in that directory. + +## Adding new tests + +The `update-dump` target will also generate new output directories for any test that does not +currently have an output directory. The output directory will contain all the output files that +the test can generate. You might not wish to test all the phases with this test. Remove any files +you do not wish to test, and commit the rest. + +## To add a new tool + +If you need to add a new tool, or a new invocation of a tool, then you create a new top-level +directory, and create the `[directoryname].cmake` file to find tests and generate +the command line. \ No newline at end of file diff --git a/testsuite/compare.cmake b/testsuite/compare.cmake new file mode 100644 index 000000000..7875e88e3 --- /dev/null +++ b/testsuite/compare.cmake @@ -0,0 +1,32 @@ +# This is used to compare to files using CMake. +# It improves on the default behaviour to print the +# the files if there is a difference. + +execute_process( + COMMAND ${CMAKE_COMMAND} -E compare_files --ignore-eol ${original_file} ${new_file} + RESULT_VARIABLE status +) + +if (${status} EQUAL 1) + message ("Compare ${original_file} with ${new_file}") + if (diff_tool STREQUAL "") + file(READ ${original_file} original_text) + file(READ ${new_file} new_text) + message("--Original File-----------------------------------------------------------------") + if (NOT original_text STREQUAL "") + message("${original_text}") + endif() + message("--------------------------------------------------------------------------------") + message(" ") + message("--New File----------------------------------------------------------------------") + if (NOT new_text STREQUAL "") + message(${new_text}) + endif() + message("--------------------------------------------------------------------------------") + else () + execute_process( + COMMAND ${diff_tool} ${original_file} ${new_file} + ) + endif () + message(FATAL_ERROR "Files differ!") +endif () \ No newline at end of file diff --git a/testsuite/run_command.cmake b/testsuite/run_command.cmake new file mode 100644 index 000000000..c613afc78 --- /dev/null +++ b/testsuite/run_command.cmake @@ -0,0 +1,35 @@ +# This is used to run a command that can fail. +# Dumping all the output and error code into a file +# Also handles timeouts + +file(REMOVE_RECURSE ${OUTPUT_DIR}) + +make_directory(${OUTPUT_DIR}) + +include(${CMAKE_CURRENT_LIST_DIR}/${TOOLNAME}.cmake) + +message("Verona Local dist: ${VERONA_LOCAL_DIST}") +toolinvoke(TOOLINVOKE ${VERONA_LOCAL_DIST} ${TESTFILE} ${OUTPUT_DIR}) + +list(JOIN TOOLINVOKE " " TOOLINVOKE_SEP) +message ("Running") +message (" ${TOOLINVOKE_SEP}") +message ("in working directory") +message (" ${WORKING_DIR}") +message ("output sent to") +message (" ${OUTPUT_DIR}") + +# Run command +execute_process( + COMMAND ${TOOLINVOKE} + WORKING_DIRECTORY ${WORKING_DIR} + OUTPUT_FILE ${OUTPUT_DIR}/stdout.txt + ERROR_FILE ${OUTPUT_DIR}/stderr.txt + TIMEOUT 20 # Timeout at 20 seconds, may need to increase this. + RESULT_VARIABLE status +) + +# Push exit code into dump and make sure both stdout and stderr exist +file(WRITE ${OUTPUT_DIR}/exit_code.txt ${status}) +file(TOUCH ${OUTPUT_DIR}/stdout.txt) +file(TOUCH ${OUTPUT_DIR}/stderr.txt) diff --git a/testsuite/verona.cmake b/testsuite/verona.cmake new file mode 100644 index 000000000..eff795437 --- /dev/null +++ b/testsuite/verona.cmake @@ -0,0 +1,7 @@ +# Arguments for testing verona-parser +macro(toolinvoke ARGS local_dist testfile outputdir) + set(${ARGS} "${local_dist}/verona/verona${CMAKE_EXECUTABLE_SUFFIX}" build ${testfile} --wf-check -o ${outputdir}/ast) +endmacro() + +# Set extension for verona tests +set(TEST_EXTENSION "verona") diff --git a/testsuite/verona/basic/ex1.verona b/testsuite/verona/basic/ex1.verona new file mode 100755 index 000000000..fc3e9508b --- /dev/null +++ b/testsuite/verona/basic/ex1.verona @@ -0,0 +1,5 @@ +class None { + create() : None { + new () + } +} \ No newline at end of file diff --git a/testsuite/verona/basic/ex1_out/exit_code.txt b/testsuite/verona/basic/ex1_out/exit_code.txt new file mode 100644 index 000000000..c22708346 --- /dev/null +++ b/testsuite/verona/basic/ex1_out/exit_code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/testsuite/verona/basic/ex1_out/stderr.txt b/testsuite/verona/basic/ex1_out/stderr.txt new file mode 100644 index 000000000..e69de29bb diff --git a/testsuite/verona/basic/ex1_out/stdout.txt b/testsuite/verona/basic/ex1_out/stdout.txt new file mode 100644 index 000000000..e69de29bb