CI #1921
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| #push: | |
| workflow_dispatch: | |
| workflow_call: | |
| pull_request_review: | |
| types: [submitted] | |
| permissions: write-all | |
| jobs: | |
| #Sanity: | |
| #uses: speedb-io/speedb/.github/workflows/sanity_check.yml@main | |
| Check-Licence-And-History: | |
| #if: ${{ github.event_name == 'pull_request_review' }} | |
| uses: ./.github/workflows/check_license_and_history.yml | |
| Build: | |
| needs: [Check-Licence-And-History] | |
| if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }} | |
| #runs-on: [self-hosted, ubuntu, asrunner] | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| include: | |
| - name: verify build | |
| command: cmake .. -GNinja | |
| - name: optimized build | |
| command: cmake .. -DCMAKE_BUILD_TYPE=Release -GNinja | |
| - mame: clang build | |
| command: CC=clang CXX=clang++ cmake .. -GNinja | |
| container: | |
| image: alpine:3.14 | |
| steps: | |
| - name: Pre-build | |
| run: | | |
| env | |
| rm -rf /usr/share/dotnet || echo "" | |
| df -h | |
| apk add git | |
| echo "nameserver 8.8.8.8" > /etc/resolv.conf | |
| apk add bash python3 py3-pip clang clang-extra-tools shellcheck gcc g++ cmake ninja ccache \ | |
| openjdk10 gflags-dev snappy-dev lz4-dev bzip2-dev zstd-dev zlib-dev linux-headers openssh-client tar readline-dev | |
| python3 -m pip install lint-diffs flake8 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - run: git config --system --add safe.directory /__w/speedb/speedb | |
| - name: Prepare ccache timestamp | |
| id: ccache_cache_timestamp | |
| shell: cmake -P {0} | |
| run: | | |
| string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) | |
| message("::set-output name=timestamp::${current_date}") | |
| - name: ccache cache files | |
| uses: actions/cache@v3.3.2 | |
| with: | |
| path: ~/.ccache | |
| key: ${{runner.os}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}} | |
| restore-keys: | | |
| ${{runner.os}}-ccache- | |
| - name: ${{ matrix.name }} | |
| run: | | |
| #echo "try git config" | |
| #git config --global --list | |
| #echo "done git config" | |
| if [ -d "$GITHUB_WORKSPACE/build" ]; then | |
| echo >&2 "error: the build directory should not exist" && false NIK | |
| fi | |
| if [ -d "~/.ccache" ]; then | |
| echo "Already exists" | |
| else | |
| mkdir -p ~/.ccache | |
| ls ~ | grep cache || echo "" | |
| touch ~/.ccache/ccache.txt | |
| echo "aaa" > ~/.ccache/ccache.txt | |
| ls ~/.ccache | |
| cat ~/.ccache/ccache.txt | |
| fi | |
| mkdir -p "$GITHUB_WORKSPACE/build" | |
| cd "$GITHUB_WORKSPACE/build" | |
| export "CCACHE_BASEDIR=$HOME" | |
| export "CCACHE_DIR=$HOME/.ccache" | |
| export "CCACHE_COMPILERCHECK=content" | |
| echo "MAtrix command - " ${{ matrix.command }} | |
| ${{ matrix.command }} -DPORTABLE=1 -DWITH_GFLAGS=1 \ | |
| -DWITH_ZLIB=1 -DWITH_SNAPPY=1 -DWITH_BZ2=1 -DWITH_LZ4=1 -DWITH_ZSTD=1 \ | |
| -DWITH_JNI=1 -DJAVA_HOME=/usr/lib/jvm/default-jvm \ | |
| -DWITH_BENCHMARK_TOOLS=1 -DWITH_CORE_TOOLS=1 -DWITH_TOOLS=1 \ | |
| -DWITH_TESTS=1 -DWITH_ALL_TESTS=1 -DWITH_EXAMPLES=1 | |
| echo "starting ninja" | |
| ninja | |
| #Performance: | |
| #if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }} | |
| #needs: [Build] | |
| #uses: speedb-io/speedb/.github/workflows/perf-test.yml@main | |
| QA-Tests: | |
| if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }} | |
| needs: [Build] | |
| uses: speedb-io/speedb/.github/workflows/qa-tests.yml@main | |
| Fuzz: | |
| if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }} | |
| needs: [Build] | |
| uses: ./.github/workflows/test_fuzz.yml | |
| Windows-build-test: | |
| if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }} | |
| needs: [Build] | |
| runs-on: windows-2022 | |
| env: | |
| THIRDPARTY_HOME: C:/Users/runneradmin/thirdparty | |
| CMAKE_HOME: C:/Program Files/CMake | |
| CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe | |
| SNAPPY_HOME: C:/Users/runneradmin/thirdparty/snappy-1.1.9 | |
| SNAPPY_INCLUDE: C:/Users/runneradmin/thirdparty/snappy-1.1.9;C:/Users/circleci/thirdparty/snappy-1.1.9/build | |
| SNAPPY_LIB_DEBUG: C:/Users/runneradmin/thirdparty/snappy-1.1.9/build/Debug/snappy.lib | |
| CMAKE_GENERATOR: Visual Studio 17 2022 | |
| CODE_HOME: C:/Users/runneradmin/code | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' # See 'Supported distributions' for available options | |
| java-version: '8' | |
| - name: Add msbuild to PATH | |
| uses: microsoft/setup-msbuild@v1.1 | |
| - name: Setup VS Dev | |
| uses: seanmiddleditch/gha-setup-vsdevenv@v4 | |
| - name: install cmake | |
| shell: powershell | |
| run: | | |
| echo "Installing CMake..." | |
| choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y | |
| - name: prepare Thirdparty home | |
| shell: powershell | |
| run: mkdir "$Env:THIRDPARTY_HOME" | |
| - name: install snappy test | |
| shell: powershell | |
| run: | | |
| mkdir $env:CODE_HOME | |
| cd $env:CODE_HOME | |
| curl https://github.com/google/snappy/archive/refs/tags/1.1.9.zip -o 1.1.9.zip | |
| Expand-Archive -Path 1.1.9.zip -DestinationPath snappy-tmp | |
| mv .\snappy-tmp\snappy-1.1.9\ . | |
| rmdir .\snappy-tmp\ | |
| cd .\snappy-1.1.9\ | |
| mkdir build | |
| cd .\build\ | |
| cmake -G "Visual Studio 17 2022" -A x64 -DCMAKE_GENERATOR_PLATFORM=x64 -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF .. | |
| msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64 | |
| msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64 | |
| - name: install snappy | |
| shell: powershell | |
| run: | | |
| cd $Env:THIRDPARTY_HOME | |
| curl https://github.com/google/snappy/archive/refs/tags/1.1.9.zip -O snappy-1.1.9.zip | |
| Expand-Archive -Path snappy-1.1.9.zip -DestinationPath snappy-tmp | |
| mv .\snappy-tmp\snappy-1.1.9\ . | |
| cd snappy-1.1.9 | |
| mkdir build | |
| cd .\build | |
| & cmake -G "Visual Studio 17 2022" -A x64 -DCMAKE_GENERATOR_PLATFORM=x64 -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF .. | |
| msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Release -property:Platform=x64 | |
| - name: install gflags | |
| shell: powershell | |
| run: | | |
| cd $Env:THIRDPARTY_HOME | |
| curl https://github.com/gflags/gflags/archive/refs/tags/v2.2.2.zip -o v2.2.2.zip | |
| Expand-Archive -Path .\v2.2.2.zip -DestinationPath gflags-tmp | |
| mv .\gflags-tmp\gflags-2.2.2 . | |
| rmdir gflags-tmp | |
| cd gflags-2.2.2 | |
| mkdir target | |
| cd target | |
| cmake -G "Visual Studio 17 2022" -A x64 .. | |
| msbuild gflags.sln /p:Configuration=Release /p:Platform=x64 | |
| - name: install zlib | |
| shell: powershell | |
| run: | | |
| cd $Env:THIRDPARTY_HOME | |
| git clone https://github.com/madler/zlib.git -b v1.3 zlib-1.3 | |
| cd zlib-1.3\contrib\vstudio\vc14 | |
| devenv zlibvc.sln /upgrade | |
| cp ../../../zlib.h . | |
| msbuild zlibvc.sln /p:Configuration=Debug /p:Platform=x64 | |
| msbuild zlibvc.sln /p:Configuration=Release /p:Platform=x64 | |
| copy x64\ZlibDllRelease\zlibwapi.lib x64\ZlibStatRelease\ | |
| - name: install lz4 | |
| shell: powershell | |
| run: | | |
| cd $Env:THIRDPARTY_HOME | |
| curl https://github.com/lz4/lz4/archive/refs/tags/v1.9.2.zip -o lz4.zip | |
| Expand-Archive -Path lz4.zip -DestinationPath lz4-tmp | |
| mv .\lz4-tmp\lz4-1.9.2\ . | |
| rmdir .\lz4-tmp\ | |
| cd .\lz4-1.9.2\ | |
| cd visual\VS2017 | |
| devenv lz4.sln /upgrade | |
| msbuild lz4.sln /p:Configuration=Release /p:Platform=x64 | |
| - name: install zctd | |
| shell: powershell | |
| run: | | |
| cd $Env:THIRDPARTY_HOME | |
| curl https://github.com/facebook/zstd/archive/v1.5.2.zip -o zstd-tmp.zip | |
| Expand-Archive -Path zstd-tmp.zip -DestinationPath zstd-tmp | |
| mv .\zstd-tmp\zstd-1.5.2\ . | |
| rmdir .\zstd-tmp\ | |
| cd zstd-1.5.2\build\VS2010 | |
| devenv zstd.sln /upgrade | |
| msbuild zstd.sln /p:Configuration=Debug /p:Platform=x64 | |
| msbuild zstd.sln /p:Configuration=Release /p:Platform=x64 | |
| - name: Build Speedb | |
| run: | | |
| copy C:\Users\runneradmin\thirdparty\snappy-1.1.9\build\snappy-stubs-public.h C:\Users\runneradmin\thirdparty\snappy-1.1.9\ | |
| copy tools\thirdparty.txt thirdparty.inc # copy the thirdparty.inc that reflects the env on the runner machine | |
| mkdir build | |
| cd build | |
| & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20 -A x64 -DJNI=1 -DGFLAGS=1 -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 -DSNAPPY=1 -DLZ4=1 -DZLIB=1 -DZSTD=1 -DXPRESS=1 -DFAIL_ON_WARNINGS=0 .. | |
| cd .. | |
| echo "Building with VS version: $Env:CMAKE_GENERATOR" | |
| msbuild build/speedb.sln /p:Configuration=Release /t:speedbjni-shared | |
| #msbuild.exe build/speedb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 | |
| - name: Test Speedb | |
| shell: powershell | |
| run: | | |
| echo "skipping all tests for now" | |
| #build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16 | |
| #build_tools\run_ci_db_test.ps1 -SuiteRun db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16 | |
| Macos-build: | |
| if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }} | |
| needs: [Build] | |
| runs-on: macos-11 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' # See 'Supported distributions' for available options | |
| java-version: '8' | |
| - name: build jar | |
| run: | | |
| echo $JAVA_HOME | |
| export CPPFLAGS="-I$JAVA_HOME/include" | |
| export CXXFLAGS="-I$JAVA_HOME/include" | |
| brew install zlib | |
| brew install bzip2 lz4 snappy | |
| ROCKSDB_DISABLE_JEMALLOC=1 PORTABLE=1 DEBUG_LEVEL=0 make -j 4 rocksdbjavastatic | |
| Linux-build-and-UnitTest: | |
| if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }} | |
| needs: [Build] | |
| runs-on: ubuntu-latest | |
| container: | |
| image: centos:7.9.2009 | |
| steps: | |
| - name: pre | |
| run: | | |
| yum install -y centos-release-scl epel-release | |
| yum install -y make devtoolset-11-gcc-c++ \ | |
| coreutils wget unzip which git python3 openssl openssl-devel \ | |
| libzstd-devel lz4-devel snappy-devel zlib-devel readline-devel \ | |
| java-1.8.0-openjdk-devel | |
| echo "PATH=/opt/rh/devtoolset-11/root/usr/bin:${PATH}" >> $GITHUB_ENV | |
| echo "RELEASE_VERSION=${GITHUB_REF_NAME#speedb/v}" >> $GITHUB_ENV | |
| - name: Install CMake | |
| run: | | |
| CMAKE_RELEASE=3.20.1 | |
| wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_RELEASE}/cmake-${CMAKE_RELEASE}.tar.gz | |
| tar xf cmake-${CMAKE_RELEASE}.tar.gz | |
| cd cmake-${CMAKE_RELEASE} | |
| ./bootstrap | |
| make -j$(nproc) && make install | |
| - uses: actions/checkout@v3 | |
| - run: mkdir "$GITHUB_WORKSPACE/out" | |
| - name: Build and package release libraries | |
| run: | | |
| rm -rf build && mkdir build && cd build | |
| cmake .. -DCMAKE_BUILD_TYPE=Release -DSPDB_RELEASE_BUILD=1 -DPORTABLE=1 -DWITH_GFLAGS=0 -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1 | |
| - name: Build Jar | |
| run: | | |
| make clean | |
| SPDB_RELEASE_BUILD=1 LIB_MODE=static DEBUG_LEVEL=0 PORTABLE=1 JAVA_HOME=/usr/lib/jvm/java-openjdk make -j$(nproc) rocksdbjavastatic | |
| - name: Java Unit test | |
| run: | | |
| make clean | |
| JAVA_HOME=/usr/lib/jvm/java-openjdk make -j$(nproc) jtest | |
| - name: Build db_bench | |
| run: | | |
| yum install -y gflags-devel | |
| rm -rf build && mkdir build && cd build | |
| cmake .. -DCMAKE_BUILD_TYPE=Release -DSPDB_RELEASE_BUILD=1 -DPORTABLE=1 -DWITH_GFLAGS=1 \ | |
| -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1 \ | |
| -DWITH_BENCHMARK_TOOLS=1 -DROCKSDB_BUILD_SHARED=1 | |
| make -j$(nproc) db_bench | |
| Linux-Arm-build: | |
| if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }} | |
| needs: [Build] | |
| runs-on: ubuArm64G | |
| steps: | |
| - name: Dump GitHub context | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| run: | | |
| echo "$GITHUB_CONTEXT" | |
| - name: 'Checkout GitHub Action' | |
| uses: actions/checkout@v3 | |
| - name: 'Build_on_Arm_Centos' | |
| run: | | |
| export SPDB_LIB_DIR=~/spdb_lib && mkdir -p $SPDB_LIB_DIR | |
| case "$GITHUB_EVENT_NAME" in | |
| "pull_request") | |
| echo "this workflow was triggered by a pull request" | |
| echo "the branch can not be used, it is $GITHUB_REF_NAME , instead ${{ github.head_ref }} will be used" | |
| docker run --rm -v $(readlink -f ${SPDB_LIB_DIR}):/out -i speedb-centos-builder ${{ github.head_ref }} | |
| ;; | |
| "workflow_dispatch") | |
| echo "this workflow was triggered by a workflow dispatch, we will use the ref_name instead of the merge branch" | |
| echo " consider using github.ref, the branch that will be used here: " ${{ github.ref_name }} | |
| docker run --rm -v $(readlink -f ${SPDB_LIB_DIR}):/out -i speedb-centos-builder ${{ github.ref_name }} | |
| ;; | |
| "pull_request_review") | |
| echo "this workflow was triggered by a pull request review" | |
| echo "the branch can not be used, it is $GITHUB_REF_NAME , instead ${{ github.event.pull_request.head.ref }} will be used" | |
| docker run --rm -v $(readlink -f ${SPDB_LIB_DIR}):/out -i speedb-centos-builder ${{ github.event.pull_request.head.ref }} | |
| ;; | |
| esac | |
| CI-all: | |
| if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }} | |
| needs: [Check-Licence-And-History, Build, QA-Tests, Fuzz, Linux-Arm-build, Linux-build-and-UnitTest, Macos-build, Windows-build-test] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Summary | |
| run: | | |
| echo "All tests passed" | |
| exit 0 |