Merge pull request #166 from laurynas-biveinis/deepstate-submodule

Convert DeepState dependency to a submodule

.github/workflows/build.yml

@@ -227,7 +227,7 @@ jobs:
           if [[ $COMPILER == "gcc" ]]; then
             sudo apt-get install -y g++-10
           elif [[ $COMPILER == "clang" ]]; then
-            sudo apt-get install -y clang-11
+            sudo apt-get install -y clang-11 libc6-dev-i386
             if [[ $BUILD_TYPE == "Release" ]]; then
               sudo apt-get install -y llvm-11-dev lld-11
             fi

.gitmodules

@@ -8,3 +8,6 @@
 [submodule "3rd_party/googletest"]
 	path = 3rd_party/googletest
 	url = https://github.com/google/googletest.git
+[submodule "3rd_party/deepstate"]
+	path = 3rd_party/deepstate
+	url = https://github.com/trailofbits/deepstate.git

.muse/dependencies.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 add-apt-repository -y 'ppa:mhier/libboost-latest'
 apt-get update
-apt-get install -y boost1.74 clang-11
+apt-get install -y boost1.74 clang-11 libc6-dev-i386
 git submodule update --init

.travis.yml

@@ -35,6 +35,7 @@ os_setups:
           - clang-tools-11
           - llvm-11-dev
           - lld-11
+          - libc6-dev-i386
           - valgrind
   macos_setup: &macos
     os: osx

CMakeLists.txt

@@ -237,11 +237,33 @@ else()
   message(STATUS "Disabling LTO for Google Benchmark")
 endif()
 add_subdirectory(3rd_party/benchmark)
+
+# TODO(laurynas): fix DeepState and GCC compatibility
+if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+  include(ExternalProject)
+  ExternalProject_Add(3rd_party_deepstate
+    SOURCE_DIR "${CMAKE_SOURCE_DIR}/3rd_party/deepstate"
+    BINARY_DIR "${CMAKE_BINARY_DIR}/3rd_party/deepstate"
+    CMAKE_ARGS "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
+    "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" "-DCMAKE_C_FLAGS=-w"
+    "-DCMAKE_CXX_FLAGS=-w"
+    INSTALL_COMMAND "")
+
+  add_library(deepstate STATIC IMPORTED)
+
+  ExternalProject_Get_property(3rd_party_deepstate SOURCE_DIR)
+  target_include_directories(deepstate INTERFACE "${SOURCE_DIR}/src/include/")
+  ExternalProject_Get_property(3rd_party_deepstate BINARY_DIR)
+  set_target_properties(deepstate PROPERTIES IMPORTED_LOCATION
+    "${BINARY_DIR}/libdeepstate.a")
+endif()
+
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU"
     AND CMAKE_BUILD_TYPE MATCHES "Debug")
   target_compile_definitions(benchmark PUBLIC _GLIBCXX_DEBUG
     _GLIBCXX_DEBUG_PEDANTIC)
 endif()
+
 # Add benchmark_include_dirs by target_include_directories(... SYSTEM ...)
 # before target_link_libraries so that benchmark headers are included through
 # -isystem not -I, resulting in build-breaking diagnostics.

README.md

@@ -12,7 +12,8 @@ Status](https://travis-ci.org/laurynas-biveinis/unodb.svg?branch=master)](https:
 ## Introduction
 
 Unodb is a adaptive radix tree implementation, done as my playground for various
-C++ tools and ideas. I am trying to describe some of the things I learned at my [blog](https://of-code.blogspot.com/search/label/art).
+C++ tools and ideas. I am trying to describe some of the things I learned at my
+[blog](https://of-code.blogspot.com/search/label/art).
 
 ## Requirements
 
@@ -81,15 +82,16 @@ The are three ART classes available:
 * CMake, at least 3.12
 * Guidelines Support Library for gsl::span, imported as a git submodule.
 * Boost library. Currently tested with versions 1.74 and 1.75.
-* clang-format 9.0
-* Google Test for tests, imported as a git submodule.
+* (optional) clang-format
 * (optional) lcov
 * (optional) clang-tidy
 * (optional) cppcheck
 * (optional) cpplint
 * (optional) include-what-you-use
-* (optional) [DeepState][deepstate] for fuzzing, currently working on macOS only
-* (optional) Google Benchmark for microbenchmarks.
+* Google Test for tests, bundled as a git submodule.
+* [DeepState][deepstate] for fuzzing `unodb::db`, currently building with clang
+  only, bundled.
+* (optional) Google Benchmark for microbenchmarks, bundled.
 
 ## Development
 

test/CMakeLists.txt

@@ -2,59 +2,41 @@
 
 enable_testing()
 
-# TODO(laurynas): convert DeepState dependency to a git submodule
-find_path(DEEPSTATE_HPP_PATH deepstate/DeepState.hpp)
-if(NOT DEEPSTATE_HPP_PATH)
-  message(STATUS "DeepState header not found, not building its fuzz tests")
-else()
-  message(STATUS "DeepState header found in ${DEEPSTATE_HPP_PATH}")
-endif()
-
-find_library(DEEPSTATE_LIB_PATH deepstate)
-if(NOT DEEPSTATE_LIB_PATH)
-  message(STATUS "DeepState library not found, not building its fuzz tests")
-else()
-  message(STATUS "DeepState library found in ${DEEPSTATE_LIB_PATH}")
-endif()
-
-include(CheckCXXSourceCompiles)
-
-set(CMAKE_REQUIRED_FLAGS "-fsanitize=fuzzer")
-set(CMAKE_REQUIRED_LINK_OPTIONS "-fsanitize=fuzzer")
-check_cxx_source_compiles(
-  "#include <cstddef>
-   #include <cstdint>
-   extern \"C\" int LLVMFuzzerTestOneInput(const std::uint8_t *, std::size_t) {
-     return 0;
-   }" LIBFUZZER_OK)
-
-if(LIBFUZZER_OK)
-  find_library(DEEPSTATE_LF_LIB_PATH deepstate_LF)
-  if(NOT DEEPSTATE_LF_LIB_PATH)
-    message(STATUS "libfuzzer-enabled DeepState not found, not building its fuzz tests")
-  else()
-    message(STATUS "libfuzzer-enabled DeepState found in ${DEEPSTATE_LF_LIB_PATH}")
+if(deepstate)
+  include(CheckCXXSourceCompiles)
+
+  set(CMAKE_REQUIRED_FLAGS "-fsanitize=fuzzer")
+  set(CMAKE_REQUIRED_LINK_OPTIONS "-fsanitize=fuzzer")
+  check_cxx_source_compiles(
+    "#include <cstddef>
+     #include <cstdint>
+     extern \"C\" int LLVMFuzzerTestOneInput(const std::uint8_t *, std::size_t) {
+       return 0;
+     }" LIBFUZZER_OK)
+
+  if(LIBFUZZER_OK)
+    find_library(DEEPSTATE_LF_LIB_PATH deepstate_LF)
+    if(NOT DEEPSTATE_LF_LIB_PATH)
+      message(STATUS "libfuzzer-enabled DeepState not found, not building its fuzz tests")
+    else()
+      message(STATUS "libfuzzer-enabled DeepState found in ${DEEPSTATE_LF_LIB_PATH}")
+    endif()
   endif()
-endif()
 
-if(DEEPSTATE_HPP_PATH)
-  if(DEEPSTATE_LIB_PATH)
-    set(DEEPSTATE_OK TRUE)
-  endif()
   if(DEEPSTATE_LF_LIB_PATH)
     set(DEEPSTATE_LF_OK TRUE)
   endif()
-endif()
 
-if(DO_CLANG_TIDY)
-  # TODO(laurynas): introduce subdir, move this to own .clang-tidy
-  string(CONCAT CLANG_TIDY_DISABLED_FOR_DEEPSTATE
-    "-cert-err58-cpp,"
-    "-cppcoreguidelines-avoid-non-const-global-variables," # TEST() macros
-    "-fuchsia-statically-constructed-objects,"
-    "-readability-implicit-bool-conversion") # DeepState_Bool returning int
-  set(DO_CLANG_TIDY_DEEPSTATE ${DO_CLANG_TIDY}
-    "-checks=${CLANG_TIDY_DISABLED_FOR_DEEPSTATE}")
+  if(DO_CLANG_TIDY)
+    # TODO(laurynas): introduce subdir, move this to own .clang-tidy
+    string(CONCAT CLANG_TIDY_DISABLED_FOR_DEEPSTATE
+      "-cert-err58-cpp,"
+      "-cppcoreguidelines-avoid-non-const-global-variables," # TEST() macros
+      "-fuchsia-statically-constructed-objects,"
+      "-readability-implicit-bool-conversion") # DeepState_Bool returning int
+    set(DO_CLANG_TIDY_DEEPSTATE ${DO_CLANG_TIDY}
+      "-checks=${CLANG_TIDY_DISABLED_FOR_DEEPSTATE}")
+  endif()
 endif()
 
 function(ADD_COVERAGE_TARGET)
@@ -119,65 +101,61 @@ if(COVERAGE)
   add_coverage_target(TARGET coverage DEPENDENCY tests_for_coverage)
 endif()
 
-function(COMMON_DEEPSTATE_TARGET_PROPERTIES TARGET)
-  common_target_properties(${TARGET})
-  target_include_directories(${TARGET} SYSTEM PRIVATE "${DEEPSTATE_HPP_PATH}")
-  target_link_libraries(${TARGET} PRIVATE unodb)
-  set_clang_tidy_options(${TARGET} "${DO_CLANG_TIDY_DEEPSTATE}")
-endfunction()
+if(deepstate)
+  function(COMMON_DEEPSTATE_TARGET_PROPERTIES TARGET)
+    common_target_properties(${TARGET})
+    target_include_directories(${TARGET} SYSTEM PRIVATE
+      "${DEEPSTATE_INCLUDE_PATH}")
+    target_link_libraries(${TARGET} PRIVATE unodb deepstate)
+    set_clang_tidy_options(${TARGET} "${DO_CLANG_TIDY_DEEPSTATE}")
+  endfunction()
 
-if(DEEPSTATE_OK)
   add_executable(test_art_fuzz_deepstate test_art_fuzz_deepstate.cpp)
   add_test(NAME test_art_fuzz_deepstate COMMAND test_art_fuzz_deepstate)
   common_deepstate_target_properties(test_art_fuzz_deepstate)
-  target_link_libraries(test_art_fuzz_deepstate PRIVATE "${DEEPSTATE_LIB_PATH}")
 
   add_custom_target(deepstate_1m
     ${CMAKE_COMMAND} -E make_directory deepstate_fails
-    COMMAND test_art_fuzz_deepstate --fuzz --timeout 60 --output_test_dir deepstate_fails)
+    COMMAND test_art_fuzz_deepstate --fuzz --timeout 60 --output_test_dir
+    deepstate_fails)
 
   add_custom_target(deepstate_20m
     ${CMAKE_COMMAND} -E make_directory deepstate_fails
-    COMMAND test_art_fuzz_deepstate --fuzz --timeout 1200 --output_test_dir deepstate_fails)
+    COMMAND test_art_fuzz_deepstate --fuzz --timeout 1200 --output_test_dir
+    deepstate_fails)
 
   add_custom_target(deepstate_8h
     ${CMAKE_COMMAND} -E make_directory deepstate_fails
-    COMMAND test_art_fuzz_deepstate --fuzz --timeout 28800 --output_test_dir deepstate_fails)
-
-  if(COVERAGE)
-    add_coverage_target(TARGET coverage_deepstate DEPENDENCY deepstate_1m)
-  endif()
-endif()
+    COMMAND test_art_fuzz_deepstate --fuzz --timeout 28800 --output_test_dir
+    deepstate_fails)
+
+  if(DEEPSTATE_LF_OK)
+    add_executable(test_art_fuzz_deepstate_lf test_art_fuzz_deepstate.cpp)
+    add_test(NAME test_art_fuzz_deepstate_lf COMMAND test_art_fuzz_deepstate_lf -runs=1)
+    common_deepstate_target_properties(test_art_fuzz_deepstate_lf)
+    set_target_properties(test_art_fuzz_deepstate_lf PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
+    target_compile_options(test_art_fuzz_deepstate_lf PRIVATE "-fsanitize=fuzzer")
+    target_link_libraries(test_art_fuzz_deepstate_lf PRIVATE "${DEEPSTATE_LF_LIB_PATH}")
+    target_link_libraries(test_art_fuzz_deepstate_lf PRIVATE "-fsanitize=fuzzer")
+
+    add_custom_target(deepstate_lf_1m
+      ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus
+      COMMAND env ${SANITIZER_ENV}
+      ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1
+      -detect_leaks=0  -max_total_time=60)
+
+    add_custom_target(deepstate_lf_20m
+      ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus
+      COMMAND env ${SANITIZER_ENV}
+      ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1
+      -detect_leaks=0  -max_total_time=1200)
+
+    add_custom_target(deepstate_lf_8h
+      ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus
+      COMMAND env ${SANITIZER_ENV}
+      ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1
+      -detect_leaks=0  -max_total_time=28800)
 
-if(DEEPSTATE_LF_OK)
-  add_executable(test_art_fuzz_deepstate_lf test_art_fuzz_deepstate.cpp)
-  add_test(NAME test_art_fuzz_deepstate_lf COMMAND test_art_fuzz_deepstate_lf -runs=1)
-  common_deepstate_target_properties(test_art_fuzz_deepstate_lf)
-  set_target_properties(test_art_fuzz_deepstate_lf PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
-  target_compile_options(test_art_fuzz_deepstate_lf PRIVATE "-fsanitize=fuzzer")
-  target_link_libraries(test_art_fuzz_deepstate_lf PRIVATE "${DEEPSTATE_LF_LIB_PATH}")
-  target_link_libraries(test_art_fuzz_deepstate_lf PRIVATE "-fsanitize=fuzzer")
-
-  add_custom_target(deepstate_lf_1m
-    ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus
-    COMMAND env ${SANITIZER_ENV}
-    ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1
-    -detect_leaks=0  -max_total_time=60)
-
-  add_custom_target(deepstate_lf_20m
-    ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus
-    COMMAND env ${SANITIZER_ENV}
-    ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1
-    -detect_leaks=0  -max_total_time=1200)
-
-  add_custom_target(deepstate_lf_8h
-    ${CMAKE_COMMAND} -E make_directory deepstate_lf_corpus
-    COMMAND env ${SANITIZER_ENV}
-    ./test_art_fuzz_deepstate_lf deepstate_lf_corpus/ -use_value_profile=1
-    -detect_leaks=0  -max_total_time=28800)
-
-  if(COVERAGE)
-    add_coverage_target(TARGET coverage_deepstate_lf DEPENDENCY deepstate_lf_1m)
   endif()
 endif()