diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 135b74de..e2601727 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: run: | sudo apt-get update sudo apt-get install cmake wget build-essential - ./install_dependencies.sh linux + ./install_dependencies.sh - name: Compile and test run: | mkdir build && cd build && cmake -G "Unix Makefiles" -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} -DCMAKE_C_COMPILER=${{ matrix.config.cc }} -DSTATIC_BUILD=1 .. @@ -67,7 +67,6 @@ jobs: run: | #brew update HOMEBREW_NO_AUTO_UPDATE=1 brew install boost automake - ./install_dependencies.sh osx - name: Compile and test run: | mkdir build && cd build && cmake -G "Unix Makefiles" -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} -DCMAKE_C_COMPILER=${{ matrix.config.cc }} .. @@ -107,7 +106,6 @@ jobs: run: | #brew update HOMEBREW_NO_AUTO_UPDATE=1 brew install boost automake libtool - ./install_dependencies.sh osx - name: Compile and test run: | mkdir build && cd build && cmake -G "Unix Makefiles" -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} -DCMAKE_C_COMPILER=${{ matrix.config.cc }} .. diff --git a/LICENSE_ODB b/LICENSE_ODB deleted file mode 100644 index 52e36215..00000000 --- a/LICENSE_ODB +++ /dev/null @@ -1,71 +0,0 @@ - -ODB License Exception for vcf-validator (Validator for the Variant Call -Format (VCF)) Software - Version 1.0 - -1. Intent - -The intent of this License Exception is to allow the vcf-validator software -to use ODB without the "copyleft" restrictions that would normally be -imposed by the GPL, the license under which ODB is distributed. - -Specifically, the intent is to allow you to use the original vcf-validator -software in your open source or proprietary works without imposing -any additional restrictions to the terms and conditions of the -Apache 2.0 license used by vcf-validator. - -If you wish to modify the vcf-validator software in such a way that it -requires changes to the persistent object model handled by ODB, you can -continue enjoying the terms of this exception provided that you are -prepared to release your modifications under Apache 2.0 or another open -source license and you obtain a separate license exception for your -modifications from Code Synthesis Tools CC. - -While this exception is specific to vcf-validator, any open source software -project licensed under a more liberal than the GPL license can obtain -a similar exception free of charge by contacting Code Synthesis Tools CC. - -2. Definitions - -"Original Software" is the vcf-validator software, Copyright (c) EMBL - -European Bioinformatics Institute, distributed under the terms of the -Apache 2.0 license, and available to any third party from: - - https://github.com/ebivariation/vcf-validator - -"ODB-Extended Software" are any modifications to the Original Software -that require additions or changes to the persistent object model of -the Original Software that is handled by ODB. - -"Work Based on the Original Software" is any derivative work under the -copyright law that is either based on the Original Software or any -modified version of the Original Software that does not constitute -ODB-Extended Software. - -"Work Based on the ODB-Extended Software" is any derivative work under -the copyright law that is based on any modified version of the Original -Software that constitutes ODB-Extended Software. - -3. Legal Terms and Conditions - -As a special exception to the terms and conditions of version 2 of the -GPL you may use the ODB runtime libraries in Original Software and in -Works Based on the Original Software and distribute the resulting works -in object code or executable form and without making source code for -such works available to any third party, provided that all of the -following conditions are met: - - a) Original Software was licensed as a whole at no charge to all third - parties under the terms of the Apache 2.0 license. - - b) The ODB runtime libraries in Works Based on the Original Software - are used solely for the purpose of supporting the functionality of - the Original Software. - -The same exception does not apply automatically to ODB-Extended Software -or Works Based on the ODB-Extended Software. If you wish to enjoy this -exception for such works, then you will need to license ODB-Extended -Software at no charge to all third parties under the terms of the Apache 2.0 -license or another open source license. You will also need to obtain a -separate License Exception for ODB-Extended Software from Code Synthesis -Tools CC. diff --git a/README.md b/README.md index 26403985..95e81687 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ Boost* | \>=1.65 | manual or automatic If you are using Ubuntu 16, you can prepare all dependencies and compile the Validation Suite with these commands: ``` sudo apt-get install cmake wget build-essential -./install_dependencies.sh linux +./install_dependencies.sh mkdir build && cd build && cmake -G "Unix Makefiles" -DSTATIC_BUILD=1 .. make ``` @@ -142,7 +142,7 @@ make The VCF Validation Suite binaries will be created in the `build/bin` subfolder. Optionally, read below for the explanation of the previous commands. ##### CMake and automatic installation -The automatic install **requires** CMake and wget to be installed before running the script (as zlib require them to be installed). Also, the script will compile some dependencies so a compilation environment is needed. If you are using Ubuntu, you can install all that with the command `sudo apt-get install cmake wget build-essential`. After installing that, use the command `./install_dependencies.sh linux`. +The automatic install **requires** CMake and wget to be installed before running the script (as zlib require them to be installed). Also, the script will compile some dependencies so a compilation environment is needed. If you are using Ubuntu, you can install all that with the command `sudo apt-get install cmake wget build-essential`. After installing that, use the command `./install_dependencies.sh`. A subfolder named `dependencies` will be created, with all the required libraries copied into it. @@ -177,7 +177,7 @@ Boost | \>=1.65 | manual You can prepare all dependencies and compile the Validation Suite with these commands: ``` -brew install cmake boost sqlite3 +brew install cmake boost mkdir build && cd build && cmake -G "Unix Makefiles" .. make ``` @@ -188,8 +188,6 @@ The VCF Validation Suite binaries will be created in the `build/bin` subfolder. The automatic install requires CMake and wget to be installed before running the script (as zlib require them to be installed). In order to set up the environment to compile the dependencies, first you need to run `brew install cmake boost`. -A subfolder named `osx_dependencies` will be created, with all the required libraries copied into it. - #### Compile In order to create the build scripts, please run `cmake` with your preferred generator. For instance, `mkdir build && cd build && cmake -G "Unix Makefiles" ..` will create Makefiles, and to build the binaries, you will need to run `make`. @@ -222,15 +220,12 @@ bootstrap In order to create the build scripts and compile vcf-validator, please run the following commands from the project root folder: ``` -cmake -DCMAKE_BUILD_TYPE=Release -G "NMake Makefiles" /path/to/CMakeLists.txt +cmake -DCMAKE_BUILD_TYPE=Release -DSTATIC_BUILD=1 -G "NMake Makefiles" /path/to/CMakeLists.txt nmake ``` Binaries will be created in the `bin` subfolder. -In order to run those binaries, you will need to add the `lib/windows_specific` directory to the `PATH`. This will allow the dll files inside that directory to be found. - - ## Deliverables The following binaries are be created after successful build: diff --git a/appveyor.yml b/appveyor.yml index c1a9fb42..f16a0db5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,8 +24,5 @@ artifacts: - path: bin\vcf_validator.exe name: vcf_validator_win.exe - - path: bin\vcf_debugulator.exe - name: vcf_debugulator_win.exe - - path: bin\vcf_assembly_checker.exe name: vcf_assembly_checker_win.exe diff --git a/docs/developer-guide.md b/docs/developer-guide.md index d33cfb88..0cd881b0 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -11,11 +11,11 @@ into the programming technical details. ## Build process overview -This is a C++ project that produces 3 binaries: vcf-validator, -vcf-assembly-checker and vcf-debugulator. The build is done with CMake (file +This is a C++ project that produces 2 binaries: vcf-validator and +vcf-assembly-checker. The build is done with CMake (file [CMakeLists.txt](../CMakeLists.txt)) for Linux, Windows and macOS. There are scripts to help installing the dependencies -([install_dependencies.sh](../install_dependencies.sh)) for Linux and macOS, and +([install_dependencies.sh](../install_dependencies.sh)) for Linux and [install_dependencies.bat](../install_dependencies.bat) for Windows). We also use TravisCI ([.travis.yml](../.travis.yml)) and AppVeyor @@ -79,11 +79,7 @@ VCF (including the metadata section), and `Error` classes (related among themselves by hierarchical inheritance) are used to represent different types of possible errors. -Based on those `Error`s, the program can produce human-readable reports and a -machine-readable report that the VCF Debugulator can read to fix some basic -errors. Later, this machine-readable report and Debugulator support were removed -to make build, packaging and installation simpler. - +Based on those `Error`s, the program can produce human-readable reports. The level of checks can be configured, from doing just a quick syntax check, to raise warnings of suspicious details. @@ -179,25 +175,9 @@ of polymorphism when we catch those exceptions in vcf.ragel (again, search for If the policy to report the errors is in use, the parser has the list of report writers the user requested. These can be any combination of -[simple report](../inc/vcf/record.hpp), [summary -report](../inc/vcf/summary_report_writer.hpp), and [ODB -report](../inc/vcf/odb_report.hpp). - -ODB is a library for Object Relational Mapping. In other words, we use it to -serialize the class data of errors, so that the debugulator can deserialize -the error details directly into variables and classes without parsing. - -We use ODB mostly in [inc/vcf/Error.hpp](../inc/vcf/error.hpp), through some -pragmas. The way ODB works is parsing the C++ classes that have ODB pragmas and -outputs C++ code that serializes the class instance information into a SQL -database. We chose Sqlite as database backend, which means that the DB reports -can actually be opened with a Sqlite client. - -The debugulator had very minimal functionality and limited usage. To reduce the -build, packaging and installation complexities, a few of the dependencies were -removed and ODB is one among them. Along with ODB, debugulator was also removed -from build as it is fully dependent on ODB. The sources and part of -documentation retained for future reference. +[simple report](../inc/vcf/record.hpp) and +[summaryreport](../inc/vcf/summary_report_writer.hpp). + ### VCF assembly checker @@ -222,31 +202,9 @@ There's also limited support to use assembly reports and translate between synonym contig names/accessions in case the nomenclature in the VCF and in the FASTA don't match. -### VCF Debugulator - -At the beginning of the project we expected there would be interesting -situations were we could use the ODB report to fix automatically wrong VCFs. -Sadly, it turned out that there's not much an automated program can do, really. - -The fixes are implemented in [inc/vcf/fixer.hpp](../inc/vcf/fixer.hpp) and -[src/vcf/fixer.cpp](../src/vcf/fixer.cpp), and basically consist on removing -duplicate variants and dropping entire fields that are non-conforming with the -spec. - -The debugulator had very minimal functionality and limited usage. To reduce the -build, packaging and installation complexities, a few of the dependencies were -removed and ODB is one among them. Along with ODB, debugulator was also removed -from build as it is fully dependent on ODB. The sources and part of -documentation retained for future reference. - ## Testing Testing is performed with [Catch](https://github.com/catchorg/Catch2) in the [`test/`](../test/) folder. The folders test/vcf/, test/fasta/ and test/assembly_report/ contain test source code, and test/input_files contain a -medium-sized set of correct and wrong VCFs that are used in the tests. - -## ODB dependency - -ODB was in use earlier and it has now been removed. Debugulator was also removed -along with it as it depends on ODB. +medium-sized set of correct and wrong VCFs that are used in the tests. \ No newline at end of file diff --git a/docs/img/VCF-validator-architecture.png b/docs/img/VCF-validator-architecture.png index b1f54e16..20bc22db 100644 Binary files a/docs/img/VCF-validator-architecture.png and b/docs/img/VCF-validator-architecture.png differ diff --git a/install_dependencies.sh b/install_dependencies.sh index 149df9d3..fa4c51c4 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -9,52 +9,31 @@ trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG trap 'if [ $? -ne 0 ]; then echo "******ERROR******: \"${last_command}\" command failed with exit code $?." && echo "Installation of dependencies failed!"; fi' EXIT help_install_dependencies="Usage: -./install_dependencies.sh [os_name] default OS is linux -./install_dependencies.sh --help displays help - -os_name can be: - - linux - - osx +./install_dependencies.sh installs dependencies for Linux OS it installs the given dependencies: - - bzip library (linux only) bzip2-1.0.6 - - zlib library (linux only) zlib-1.2.11 - - curl library (linux only) curl-7.62.0 - - openssl library (linux only) openssl-1.1.1w - - c-ares library (linux only) c-ares-1.15.0 - - boost library (linux only) boost-1.72.0 - -for linux: -./install_dependencies.sh linux - -for mac os: -./install_dependencies.sh osx + - bzip library bzip2-1.0.6 + - zlib library zlib-1.2.11 + - curl library curl-7.62.0 + - openssl library openssl-1.1.1w + - c-ares library c-ares-1.15.0 + - boost library boost-1.72.0 " - -if [ "$#" -eq 0 ] -then - OS_NAME="linux" -elif [ "$#" -eq 1 ] +if [ "$#" -ne 0 ] then - if [[ "$1" == "--help" ]] - then - echo "$help_install_dependencies" - exit - elif [[ "$1" == "linux" ]] - then - OS_NAME="linux" - elif [[ "$1" == "osx" ]] + echo "$help_install_dependencies" + exit 1 +else + OS_NAME=`uname` + if [[ $OS_NAME != "Linux" ]] then - OS_NAME="osx" - else + echo " +This script is to be executed only for Linux. +" echo "$help_install_dependencies" - exit + exit 1 fi -else - echo "requires 1 argument at max" - echo "$help_install_dependencies" - exit fi dependencies_dir="dependencies" @@ -75,81 +54,76 @@ dependencies_dir_abs_path=`pwd` build_dir_abs_path=${dependencies_dir_abs_path}/build mkdir $build_dir_abs_path -if [[ "$OS_NAME" != "osx" ]] -then - #linux - - # Download libbz2 and libz. - echo "installing libbz2" - wget http://archive.ubuntu.com/ubuntu/pool/main/b/bzip2/bzip2_1.0.6.orig.tar.bz2 -O ./libbz2.tar.bz2 - tar jxf ./libbz2.tar.bz2 - cd bzip2-1.0.6 && make && make install PREFIX=$build_dir_abs_path - cd .. - - echo "installing libz" - wget http://prdownloads.sourceforge.net/libpng/zlib-1.2.11.tar.gz?download -O ./libz.tar.gz - tar zxf ./libz.tar.gz - cd zlib-1.2.11 && ./configure --prefix=$build_dir_abs_path && make install - cd .. - - echo "installing openssl" - #using openssl v1.1.1w which supports mac arm as well - wget https://www.openssl.org/source/openssl-1.1.1w.tar.gz -O ./openssl-1.1.1w.tar.gz - tar xzf ./openssl-1.1.1w.tar.gz - cd openssl-1.1.1w - LIBS="-lcrypto -ldl" \ - ./config -static --static -fPIC no-shared no-threads \ - --prefix=$build_dir_abs_path \ - --openssldir=$build_dir_abs_path - make && make install_sw - cd .. - - echo "installing c-ares" - wget https://github.com/c-ares/c-ares/releases/download/cares-1_15_0/c-ares-1.15.0.tar.gz -O ./c-ares-1.15.0.tar.gz - tar xzf ./c-ares-1.15.0.tar.gz - cd c-ares-1.15.0 - ./configure --prefix=$build_dir_abs_path - make && make install - cd .. - - echo "installing libcurl" - wget https://curl.haxx.se/download/curl-7.62.0.tar.gz -O ./curl-7.62.0.tar.gz - tar zxf ./curl-7.62.0.tar.gz - cd curl-7.62.0 - LDFLAGS="-L$build_dir_abs_path/lib" \ - CPPFLAGS="-I$build_dir_abs_path/include" \ - ./configure --disable-shared \ - --enable-static \ - --without-librtmp \ - --without-ca-bundle \ - --disable-ldap \ - --without-zlib \ - --without-libidn2 \ - --without-nss \ - --enable-ares=$build_dir_abs_path \ - --with-ssl=$build_dir_abs_path \ - --prefix=$build_dir_abs_path - make && make install - cd .. - - echo "installing boost" - wget https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz -O ./boost.tar.gz - tar zxf ./boost.tar.gz - mv boost_1_72_0 boost && cd boost - - # If there's some error like "undefined reference to bzip2_base", then boost didn't compile everything for iostreams. - # The boost iostreams compilation was ok if this command finds some lines "nm libboost_iostreams.a | grep bzip2_base". - # The next user-config.jam should fix that. - # Documentation on building iostreams: https://www.boost.org/doc/libs/1_72_0/libs/iostreams/doc/index.html - # Documentation on specifying zlib and bzip2: https://boostorg.github.io/build/manual/master/index.html#bbv2.reference.tools.libraries.zlib - echo " +# Download libbz2 and libz. +echo "installing libbz2" +wget http://archive.ubuntu.com/ubuntu/pool/main/b/bzip2/bzip2_1.0.6.orig.tar.bz2 -O ./libbz2.tar.bz2 +tar jxf ./libbz2.tar.bz2 +cd bzip2-1.0.6 && make && make install PREFIX=$build_dir_abs_path +cd .. + +echo "installing libz" +wget http://prdownloads.sourceforge.net/libpng/zlib-1.2.11.tar.gz?download -O ./libz.tar.gz +tar zxf ./libz.tar.gz +cd zlib-1.2.11 && ./configure --prefix=$build_dir_abs_path && make install +cd .. + +echo "installing openssl" +#using openssl v1.1.1w which supports mac arm as well +wget https://www.openssl.org/source/openssl-1.1.1w.tar.gz -O ./openssl-1.1.1w.tar.gz +tar xzf ./openssl-1.1.1w.tar.gz +cd openssl-1.1.1w +LIBS="-lcrypto -ldl" \ +./config -static --static -fPIC no-shared no-threads \ + --prefix=$build_dir_abs_path \ + --openssldir=$build_dir_abs_path +make && make install_sw +cd .. + +echo "installing c-ares" +wget https://github.com/c-ares/c-ares/releases/download/cares-1_15_0/c-ares-1.15.0.tar.gz -O ./c-ares-1.15.0.tar.gz +tar xzf ./c-ares-1.15.0.tar.gz +cd c-ares-1.15.0 +./configure --prefix=$build_dir_abs_path +make && make install +cd .. + +echo "installing libcurl" +wget https://curl.haxx.se/download/curl-7.62.0.tar.gz -O ./curl-7.62.0.tar.gz +tar zxf ./curl-7.62.0.tar.gz +cd curl-7.62.0 +LDFLAGS="-L$build_dir_abs_path/lib" \ +CPPFLAGS="-I$build_dir_abs_path/include" \ +./configure --disable-shared \ + --enable-static \ + --without-librtmp \ + --without-ca-bundle \ + --disable-ldap \ + --without-zlib \ + --without-libidn2 \ + --without-nss \ + --enable-ares=$build_dir_abs_path \ + --with-ssl=$build_dir_abs_path \ + --prefix=$build_dir_abs_path +make && make install +cd .. + +echo "installing boost" +wget https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz -O ./boost.tar.gz +tar zxf ./boost.tar.gz +mv boost_1_72_0 boost && cd boost + +# If there's some error like "undefined reference to bzip2_base", then boost didn't compile everything for iostreams. +# The boost iostreams compilation was ok if this command finds some lines "nm libboost_iostreams.a | grep bzip2_base". +# The next user-config.jam should fix that. +# Documentation on building iostreams: https://www.boost.org/doc/libs/1_72_0/libs/iostreams/doc/index.html +# Documentation on specifying zlib and bzip2: https://boostorg.github.io/build/manual/master/index.html#bbv2.reference.tools.libraries.zlib +echo " using zlib : 1.2.11 : ${dependencies_dir_abs_path}/zlib-1.2.11 ; using bzip2 : 1.0.6 : ${dependencies_dir_abs_path}/bzip2-1.0.6 ; " > tools/build/src/user-config.jam - ./bootstrap.sh --prefix=${build_dir_abs_path} --with-libraries=filesystem,iostreams,log,program_options,regex,system && ./b2 link=static --prefix=${build_dir_abs_path} install - cd .. -fi +./bootstrap.sh --prefix=${build_dir_abs_path} --with-libraries=filesystem,iostreams,log,program_options,regex,system && ./b2 link=static --prefix=${build_dir_abs_path} install +cd .. cd ..