From 59d2aeea3889cb621f3305c84516d5bf311575ff Mon Sep 17 00:00:00 2001 From: Laurynas Biveinis Date: Mon, 22 Mar 2021 08:56:11 +0200 Subject: [PATCH] Convert DeepState dependency to a submodule Use CMake ExternalProject instead of add_subdirectory due to DeepState CMake being written as a top-level CMake project only. --- .github/workflows/build.yml | 2 +- .gitmodules | 3 + .muse/dependencies.sh | 2 +- .travis.yml | 1 + 3rd_party/deepstate | 1 + CMakeLists.txt | 22 +++++ README.md | 12 +-- test/CMakeLists.txt | 162 ++++++++++++++++-------------------- 8 files changed, 106 insertions(+), 99 deletions(-) create mode 160000 3rd_party/deepstate diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c519a5d6..8c87e9e1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -227,7 +227,7 @@ jobs: if [[ $COMPILER == "gcc" ]]; then sudo apt-get install -y g++-10 elif [[ $COMPILER == "clang" ]]; then - sudo apt-get install -y clang-11 + sudo apt-get install -y clang-11 libc6-dev-i386 if [[ $BUILD_TYPE == "Release" ]]; then sudo apt-get install -y llvm-11-dev lld-11 fi diff --git a/.gitmodules b/.gitmodules index 93843fbe..0a587613 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,3 +8,6 @@ [submodule "3rd_party/googletest"] path = 3rd_party/googletest url = https://github.com/google/googletest.git +[submodule "3rd_party/deepstate"] + path = 3rd_party/deepstate + url = https://github.com/trailofbits/deepstate.git diff --git a/.muse/dependencies.sh b/.muse/dependencies.sh index cd3d03f3..ca8fb7ba 100755 --- a/.muse/dependencies.sh +++ b/.muse/dependencies.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash add-apt-repository -y 'ppa:mhier/libboost-latest' apt-get update -apt-get install -y boost1.74 clang-11 +apt-get install -y boost1.74 clang-11 libc6-dev-i386 git submodule update --init diff --git a/.travis.yml b/.travis.yml index 402ee48a..01edb7cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,7 @@ os_setups: - clang-tools-11 - llvm-11-dev - lld-11 + - libc6-dev-i386 - valgrind macos_setup: &macos os: osx diff --git a/3rd_party/deepstate b/3rd_party/deepstate new file mode 160000 index 00000000..0c66cfd9 --- /dev/null +++ b/3rd_party/deepstate @@ -0,0 +1 @@ +Subproject commit 0c66cfd934131419a9712cc955a4b12cd1dedd7d diff --git a/CMakeLists.txt b/CMakeLists.txt index 1440cb7a..63d67163 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -237,11 +237,33 @@ else() message(STATUS "Disabling LTO for Google Benchmark") endif() add_subdirectory(3rd_party/benchmark) + +# TODO(laurynas): fix DeepState and GCC compatibility +if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + include(ExternalProject) + ExternalProject_Add(3rd_party_deepstate + SOURCE_DIR "${CMAKE_SOURCE_DIR}/3rd_party/deepstate" + BINARY_DIR "${CMAKE_BINARY_DIR}/3rd_party/deepstate" + CMAKE_ARGS "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" + "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" "-DCMAKE_C_FLAGS=-w" + "-DCMAKE_CXX_FLAGS=-w" + INSTALL_COMMAND "") + + add_library(deepstate STATIC IMPORTED) + + ExternalProject_Get_property(3rd_party_deepstate SOURCE_DIR) + target_include_directories(deepstate INTERFACE "${SOURCE_DIR}/src/include/") + ExternalProject_Get_property(3rd_party_deepstate BINARY_DIR) + set_target_properties(deepstate PROPERTIES IMPORTED_LOCATION + "${BINARY_DIR}/libdeepstate.a") +endif() + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_BUILD_TYPE MATCHES "Debug") target_compile_definitions(benchmark PUBLIC _GLIBCXX_DEBUG _GLIBCXX_DEBUG_PEDANTIC) endif() + # Add benchmark_include_dirs by target_include_directories(... SYSTEM ...) # before target_link_libraries so that benchmark headers are included through # -isystem not -I, resulting in build-breaking diagnostics. diff --git a/README.md b/README.md index fd3fb281..fec8acb6 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ Status](https://travis-ci.org/laurynas-biveinis/unodb.svg?branch=master)](https: ## Introduction Unodb is a adaptive radix tree implementation, done as my playground for various -C++ tools and ideas. I am trying to describe some of the things I learned at my [blog](https://of-code.blogspot.com/search/label/art). +C++ tools and ideas. I am trying to describe some of the things I learned at my +[blog](https://of-code.blogspot.com/search/label/art). ## Requirements @@ -81,15 +82,16 @@ The are three ART classes available: * CMake, at least 3.12 * Guidelines Support Library for gsl::span, imported as a git submodule. * Boost library. Currently tested with versions 1.74 and 1.75. -* clang-format 9.0 -* Google Test for tests, imported as a git submodule. +* (optional) clang-format * (optional) lcov * (optional) clang-tidy * (optional) cppcheck * (optional) cpplint * (optional) include-what-you-use -* (optional) [DeepState][deepstate] for fuzzing, currently working on macOS only -* (optional) Google Benchmark for microbenchmarks. +* Google Test for tests, bundled as a git submodule. +* [DeepState][deepstate] for fuzzing `unodb::db`, currently building with clang + only, bundled. +* (optional) Google Benchmark for microbenchmarks, bundled. ## Development diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7a09b9a6..6746e283 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -2,59 +2,41 @@ enable_testing() -# TODO(laurynas): convert DeepState dependency to a git submodule -find_path(DEEPSTATE_HPP_PATH deepstate/DeepState.hpp) -if(NOT DEEPSTATE_HPP_PATH) - message(STATUS "DeepState header not found, not building its fuzz tests") -else() - message(STATUS "DeepState header found in ${DEEPSTATE_HPP_PATH}") -endif() - -find_library(DEEPSTATE_LIB_PATH deepstate) -if(NOT DEEPSTATE_LIB_PATH) - message(STATUS "DeepState library not found, not building its fuzz tests") -else() - message(STATUS "DeepState library found in ${DEEPSTATE_LIB_PATH}") -endif() - -include(CheckCXXSourceCompiles) - -set(CMAKE_REQUIRED_FLAGS "-fsanitize=fuzzer") -set(CMAKE_REQUIRED_LINK_OPTIONS "-fsanitize=fuzzer") -check_cxx_source_compiles( - "#include - #include - extern \"C\" int LLVMFuzzerTestOneInput(const std::uint8_t *, std::size_t) { - return 0; - }" LIBFUZZER_OK) - -if(LIBFUZZER_OK) - find_library(DEEPSTATE_LF_LIB_PATH deepstate_LF) - if(NOT DEEPSTATE_LF_LIB_PATH) - message(STATUS "libfuzzer-enabled DeepState not found, not building its fuzz tests") - else() - message(STATUS "libfuzzer-enabled DeepState found in ${DEEPSTATE_LF_LIB_PATH}") +if(deepstate) + include(CheckCXXSourceCompiles) + + set(CMAKE_REQUIRED_FLAGS "-fsanitize=fuzzer") + set(CMAKE_REQUIRED_LINK_OPTIONS "-fsanitize=fuzzer") + check_cxx_source_compiles( + "#include + #include + extern \"C\" int LLVMFuzzerTestOneInput(const std::uint8_t *, std::size_t) { + return 0; + }" LIBFUZZER_OK) + + if(LIBFUZZER_OK) + find_library(DEEPSTATE_LF_LIB_PATH deepstate_LF) + if(NOT DEEPSTATE_LF_LIB_PATH) + message(STATUS "libfuzzer-enabled DeepState not found, not building its fuzz tests") + else() + message(STATUS "libfuzzer-enabled DeepState found in ${DEEPSTATE_LF_LIB_PATH}") + endif() endif() -endif() -if(DEEPSTATE_HPP_PATH) - if(DEEPSTATE_LIB_PATH) - set(DEEPSTATE_OK TRUE) - endif() if(DEEPSTATE_LF_LIB_PATH) set(DEEPSTATE_LF_OK TRUE) endif() -endif() -if(DO_CLANG_TIDY) - # TODO(laurynas): introduce subdir, move this to own .clang-tidy - string(CONCAT CLANG_TIDY_DISABLED_FOR_DEEPSTATE - "-cert-err58-cpp," - "-cppcoreguidelines-avoid-non-const-global-variables," # TEST() macros - "-fuchsia-statically-constructed-objects," - "-readability-implicit-bool-conversion") # DeepState_Bool returning int - set(DO_CLANG_TIDY_DEEPSTATE ${DO_CLANG_TIDY} - "-checks=${CLANG_TIDY_DISABLED_FOR_DEEPSTATE}") + if(DO_CLANG_TIDY) + # TODO(laurynas): introduce subdir, move this to own .clang-tidy + string(CONCAT CLANG_TIDY_DISABLED_FOR_DEEPSTATE + "-cert-err58-cpp," + "-cppcoreguidelines-avoid-non-const-global-variables," # TEST() macros + "-fuchsia-statically-constructed-objects," + "-readability-implicit-bool-conversion") # DeepState_Bool returning int + set(DO_CLANG_TIDY_DEEPSTATE ${DO_CLANG_TIDY} + "-checks=${CLANG_TIDY_DISABLED_FOR_DEEPSTATE}") + endif() endif() function(ADD_COVERAGE_TARGET) @@ -119,65 +101,61 @@ if(COVERAGE) add_coverage_target(TARGET coverage DEPENDENCY tests_for_coverage) endif() -function(COMMON_DEEPSTATE_TARGET_PROPERTIES TARGET) - common_target_properties(${TARGET}) - target_include_directories(${TARGET} SYSTEM PRIVATE "${DEEPSTATE_HPP_PATH}") - target_link_libraries(${TARGET} PRIVATE unodb) - set_clang_tidy_options(${TARGET} "${DO_CLANG_TIDY_DEEPSTATE}") -endfunction() +if(deepstate) + function(COMMON_DEEPSTATE_TARGET_PROPERTIES TARGET) + common_target_properties(${TARGET}) + target_include_directories(${TARGET} SYSTEM PRIVATE + "${DEEPSTATE_INCLUDE_PATH}") + target_link_libraries(${TARGET} PRIVATE unodb deepstate) + set_clang_tidy_options(${TARGET} "${DO_CLANG_TIDY_DEEPSTATE}") + endfunction() -if(DEEPSTATE_OK) add_executable(test_art_fuzz_deepstate test_art_fuzz_deepstate.cpp) add_test(NAME test_art_fuzz_deepstate COMMAND test_art_fuzz_deepstate) common_deepstate_target_properties(test_art_fuzz_deepstate) - target_link_libraries(test_art_fuzz_deepstate PRIVATE "${DEEPSTATE_LIB_PATH}") add_custom_target(deepstate_1m ${CMAKE_COMMAND} -E make_directory deepstate_fails - COMMAND test_art_fuzz_deepstate --fuzz --timeout 60 --output_test_dir deepstate_fails) + COMMAND test_art_fuzz_deepstate --fuzz --timeout 60 --output_test_dir + deepstate_fails) add_custom_target(deepstate_20m ${CMAKE_COMMAND} -E make_directory deepstate_fails - COMMAND test_art_fuzz_deepstate --fuzz --timeout 1200 --output_test_dir deepstate_fails) + COMMAND test_art_fuzz_deepstate --fuzz --timeout 1200 --output_test_dir + deepstate_fails) add_custom_target(deepstate_8h ${CMAKE_COMMAND} -E make_directory deepstate_fails - COMMAND test_art_fuzz_deepstate --fuzz --timeout 28800 --output_test_dir deepstate_fails) - - if(COVERAGE) - add_coverage_target(TARGET coverage_deepstate DEPENDENCY deepstate_1m) - endif() -endif() + COMMAND test_art_fuzz_deepstate --fuzz --timeout 28800 --output_test_dir + deepstate_fails) + + if(DEEPSTATE_LF_OK) + add_executable(test_art_fuzz_deepstate_lf test_art_fuzz_deepstate.cpp) + add_test(NAME test_art_fuzz_deepstate_lf COMMAND test_art_fuzz_deepstate_lf -runs=1) + common_deepstate_target_properties(test_art_fuzz_deepstate_lf) + set_target_properties(test_art_fuzz_deepstate_lf PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") + target_compile_options(test_art_fuzz_deepstate_lf PRIVATE "-fsanitize=fuzzer") + target_link_libraries(test_art_fuzz_deepstate_lf PRIVATE "${DEEPSTATE_LF_LIB_PATH}") + target_link_libraries(test_art_fuzz_deepstate_lf PRIVATE "-fsanitize=fuzzer") + + add_custom_target(deepstate_lf_1m + ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus + COMMAND env ${SANITIZER_ENV} + ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1 + -detect_leaks=0 -max_total_time=60) + + add_custom_target(deepstate_lf_20m + ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus + COMMAND env ${SANITIZER_ENV} + ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1 + -detect_leaks=0 -max_total_time=1200) + + add_custom_target(deepstate_lf_8h + ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus + COMMAND env ${SANITIZER_ENV} + ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1 + -detect_leaks=0 -max_total_time=28800) -if(DEEPSTATE_LF_OK) - add_executable(test_art_fuzz_deepstate_lf test_art_fuzz_deepstate.cpp) - add_test(NAME test_art_fuzz_deepstate_lf COMMAND test_art_fuzz_deepstate_lf -runs=1) - common_deepstate_target_properties(test_art_fuzz_deepstate_lf) - set_target_properties(test_art_fuzz_deepstate_lf PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") - target_compile_options(test_art_fuzz_deepstate_lf PRIVATE "-fsanitize=fuzzer") - target_link_libraries(test_art_fuzz_deepstate_lf PRIVATE "${DEEPSTATE_LF_LIB_PATH}") - target_link_libraries(test_art_fuzz_deepstate_lf PRIVATE "-fsanitize=fuzzer") - - add_custom_target(deepstate_lf_1m - ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus - COMMAND env ${SANITIZER_ENV} - ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1 - -detect_leaks=0 -max_total_time=60) - - add_custom_target(deepstate_lf_20m - ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus - COMMAND env ${SANITIZER_ENV} - ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1 - -detect_leaks=0 -max_total_time=1200) - - add_custom_target(deepstate_lf_8h - ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus - COMMAND env ${SANITIZER_ENV} - ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1 - -detect_leaks=0 -max_total_time=28800) - - if(COVERAGE) - add_coverage_target(TARGET coverage_deepstate_lf DEPENDENCY deepstate_lf_1m) endif() endif()