feat(backup): 2. update and refactor meta backup engine class #3482
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| name: Cpp CI | |
| on: | |
| # run on each pull request | |
| pull_request: | |
| types: [ synchronize, opened, reopened ] | |
| branches: | |
| - master | |
| - 'v[0-9]+.*' # release branch | |
| - ci-test # testing branch for github action | |
| - '*dev' | |
| paths: | |
| - .github/actions | |
| - .github/workflows/lint_and_test_cpp.yaml | |
| - .github/workflows/thirdparty-regular-push.yml | |
| - build_tools/pack_server.sh | |
| - build_tools/pack_tools.sh | |
| - docker/thirdparties-bin/** | |
| - docker/thirdparties-src/** | |
| - CMakeLists.txt | |
| - cmake_modules/** | |
| - run.sh | |
| - src/** | |
| - thirdparty/** | |
| # for manually triggering workflow | |
| workflow_dispatch: | |
| env: | |
| # Update the options to reduce the consumption of the disk space | |
| ONEBOX_OPTS: disk_min_available_space_ratio=5 | |
| TEST_OPTS: disk_min_available_space_ratio=5;throttle_test_medium_value_kb=10;throttle_test_large_value_kb=25 | |
| jobs: | |
| cpp_clang_format_linter: | |
| name: Format | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: clang-format | |
| run: ./build_tools/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . | |
| cpp_clang_tidy_linter: | |
| name: Tidy | |
| runs-on: ubuntu-22.04 | |
| container: | |
| image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }} | |
| steps: | |
| - name: Install Softwares | |
| run: | | |
| apt-get update | |
| apt-get install clang-tidy -y | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Rebuild thirdparty if needed | |
| uses: "./.github/actions/rebuild_thirdparty_if_needed" | |
| - name: clang-tidy | |
| run: | | |
| git config --global --add safe.directory $(pwd) | |
| ./run.sh build --test --compiler clang-14,clang++-14 -t debug --skip_thirdparty -c --cmake_only | |
| ./build_tools/clang_tidy.py --rev-range $(git log origin/${{ github.base_ref }} -n1 --format=format:"%H") | |
| shell: bash | |
| iwyu: | |
| name: IWYU | |
| needs: | |
| - cpp_clang_format_linter | |
| runs-on: ubuntu-latest | |
| env: | |
| USE_JEMALLOC: OFF | |
| container: | |
| image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Free Disk Space (Ubuntu) | |
| run: | | |
| .github/workflows/free_disk_space.sh | |
| - name: Rebuild thirdparty if needed | |
| uses: "./.github/actions/rebuild_thirdparty_if_needed" | |
| - name: Build IWYU binary | |
| run: | | |
| mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git | |
| cd include-what-you-use && git checkout clang_14 | |
| cd .. && mkdir build && cd build && cmake -G "Unix Makefiles" -DCMAKE_PREFIX_PATH=/usr/lib/llvm-14 ../include-what-you-use | |
| make -j $(nproc) | |
| - name: Check IWYU | |
| run: | | |
| PWD=`pwd` | |
| ./run.sh build --test --compiler clang-14,clang++-14 -j $(nproc) -t release --iwyu ${PWD}/iwyu/build/bin/include-what-you-use --skip_thirdparty | |
| python3 iwyu/include-what-you-use/fix_includes.py --blank_lines --reorder --nocomments --nosafe_headers --separate_project_includes="<tld>" --ignore_re="thrift-gen|rrdb_types|layer2_types|metrics_test.cpp" --dry_run < build/latest/iwyu.out | |
| python3 iwyu/include-what-you-use/fix_includes.py --blank_lines --reorder --nocomments --nosafe_headers --separate_project_includes="<tld>" --ignore_re="thrift-gen|rrdb_types|layer2_types|metrics_test.cpp" --dry_run < build/latest/iwyu.out | egrep "IWYU edited 0 files on your behalf" | |
| build_Release: | |
| name: Build Release | |
| needs: | |
| - cpp_clang_format_linter | |
| runs-on: ubuntu-latest | |
| env: | |
| USE_JEMALLOC: OFF | |
| ARTIFACT_NAME: release | |
| BUILD_OPTIONS: -t release --test | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| steps: | |
| - name: Clone code | |
| uses: actions/checkout@v4 | |
| - name: Rebuild thirdparty if needed | |
| uses: "./.github/actions/rebuild_thirdparty_if_needed" | |
| - name: Build Pegasus | |
| uses: "./.github/actions/build_pegasus" | |
| - name: Upload artifact | |
| uses: "./.github/actions/upload_artifact" | |
| test_Release: | |
| name: Test Release | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_module: | |
| - backup_restore_test | |
| - base_api_test | |
| - base_test | |
| - bulk_load_test | |
| # TODO(wangdan): Since the hotspot detection depends on the perf-counters system which | |
| # is being replaced with the new metrics system, its test will fail. Temporarily disable | |
| # the test and re-enable it after the hotspot detection is migrated to the new metrics | |
| # system. | |
| # - detect_hotspot_test | |
| - dsn_aio_test | |
| - dsn_block_service_test | |
| - dsn_client_test | |
| - dsn.failure_detector.tests | |
| - dsn_http_test | |
| - dsn_meta_state_tests | |
| - dsn.meta.test | |
| - dsn_nfs_test | |
| # TODO(wangdan): Since builtin_counters (memused.virt and memused.res) for perf-counters | |
| # have been removed and dsn_perf_counter_test depends on them, disable it. | |
| # - dsn_perf_counter_test | |
| - dsn_ranger_tests | |
| - dsn_replica_backup_test | |
| - dsn_replica_bulk_load_test | |
| - dsn_replica_dup_test | |
| - dsn_replica_split_test | |
| - dsn_rpc_tests | |
| - dsn.replica.test | |
| - dsn_replication_common_test | |
| - dsn.replication.simple_kv | |
| - dsn.rep_tests.simple_kv | |
| - dsn_runtime_tests | |
| - dsn_security_tests | |
| - dsn_task_tests | |
| - dsn_utils_tests | |
| - dsn.zookeeper.tests | |
| # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after | |
| # it has been optimized. | |
| # - partition_split_test | |
| - pegasus_geo_test | |
| - pegasus_rproxy_test | |
| - pegasus_unit_test | |
| - recovery_test | |
| - restore_test | |
| - security_test | |
| - throttle_test | |
| needs: build_Release | |
| runs-on: ubuntu-latest | |
| env: | |
| ARTIFACT_NAME: release | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| options: --cap-add=SYS_PTRACE | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download artifact | |
| uses: "./.github/actions/download_artifact" | |
| - name: Run server tests | |
| uses: "./.github/actions/run_server_tests" | |
| build_ASAN: | |
| name: Build ASAN | |
| needs: | |
| - cpp_clang_format_linter | |
| runs-on: ubuntu-latest | |
| env: | |
| USE_JEMALLOC: OFF | |
| ARTIFACT_NAME: release_address | |
| BUILD_OPTIONS: --sanitizer address --disable_gperf --test | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Rebuild thirdparty if needed | |
| uses: "./.github/actions/rebuild_thirdparty_if_needed" | |
| - name: Build Pegasus | |
| uses: "./.github/actions/build_pegasus" | |
| - name: Upload artifact | |
| uses: "./.github/actions/upload_artifact" | |
| test_ASAN: | |
| name: Test ASAN | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_module: | |
| - backup_restore_test | |
| - base_api_test | |
| - base_test | |
| - bulk_load_test | |
| # TODO(wangdan): Since the hotspot detection depends on the perf-counters system which | |
| # is being replaced with the new metrics system, its test will fail. Temporarily disable | |
| # the test and re-enable it after the hotspot detection is migrated to the new metrics | |
| # system. | |
| # - detect_hotspot_test | |
| - dsn_aio_test | |
| - dsn_block_service_test | |
| - dsn_client_test | |
| - dsn.failure_detector.tests | |
| - dsn_http_test | |
| - dsn_meta_state_tests | |
| - dsn.meta.test | |
| - dsn_nfs_test | |
| # TODO(wangdan): Since builtin_counters (memused.virt and memused.res) for perf-counters | |
| # have been removed and dsn_perf_counter_test depends on them, disable it. | |
| # - dsn_perf_counter_test | |
| - dsn_ranger_tests | |
| - dsn_replica_backup_test | |
| - dsn_replica_bulk_load_test | |
| - dsn_replica_dup_test | |
| - dsn_replica_split_test | |
| - dsn_rpc_tests | |
| - dsn.replica.test | |
| - dsn_replication_common_test | |
| - dsn.replication.simple_kv | |
| - dsn.rep_tests.simple_kv | |
| - dsn_runtime_tests | |
| - dsn_security_tests | |
| - dsn_task_tests | |
| - dsn_utils_tests | |
| - dsn.zookeeper.tests | |
| # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after | |
| # it has been optimized. | |
| # - partition_split_test | |
| - pegasus_geo_test | |
| - pegasus_rproxy_test | |
| - pegasus_unit_test | |
| - recovery_test | |
| - restore_test | |
| - security_test | |
| # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after | |
| # it has been optimized. | |
| # - throttle_test | |
| needs: build_ASAN | |
| runs-on: ubuntu-latest | |
| env: | |
| ARTIFACT_NAME: release_address | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| options: --cap-add=SYS_PTRACE | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download artifact | |
| uses: "./.github/actions/download_artifact" | |
| - name: Run server tests | |
| uses: "./.github/actions/run_server_tests" | |
| # TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests | |
| # when we are going to release a stable version. So we disable them in regular CI | |
| # before we find any way to reduce the time cost. | |
| # build_UBSAN: | |
| # name: Build UBSAN | |
| # needs: | |
| # - cpp_clang_format_linter | |
| # runs-on: ubuntu-latest | |
| # env: | |
| # USE_JEMALLOC: OFF | |
| # ARTIFACT_NAME: release_undefined | |
| # BUILD_OPTIONS: --sanitizer undefined --disable_gperf --test | |
| # container: | |
| # image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - name: Rebuild thirdparty if needed | |
| # uses: "./.github/actions/rebuild_thirdparty_if_needed" | |
| # - name: Build Pegasus | |
| # uses: "./.github/actions/build_pegasus" | |
| # - name: Upload artifact | |
| # uses: "./.github/actions/upload_artifact" | |
| # | |
| # test_UBSAN: | |
| # name: Test UBSAN | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # test_module: | |
| # - backup_restore_test | |
| # - base_api_test | |
| # - base_test | |
| # - bulk_load_test | |
| # # TODO(wangdan): Since the hotspot detection depends on the perf-counters system which | |
| # # is being replaced with the new metrics system, its test will fail. Temporarily disable | |
| # # the test and re-enable it after the hotspot detection is migrated to the new metrics | |
| # # system. | |
| # # - detect_hotspot_test | |
| # - dsn_aio_test | |
| # - dsn_block_service_test | |
| # - dsn_client_test | |
| # - dsn.failure_detector.tests | |
| # - dsn_http_test | |
| # - dsn_meta_state_tests | |
| # - dsn.meta.test | |
| # - dsn_nfs_test | |
| # # TODO(wangdan): Since builtin_counters (memused.virt and memused.res) for perf-counters | |
| # # have been removed and dsn_perf_counter_test depends on them, disable it. | |
| # # - dsn_perf_counter_test | |
| # - dsn_ranger_tests | |
| # - dsn_replica_backup_test | |
| # - dsn_replica_bulk_load_test | |
| # - dsn_replica_dup_test | |
| # - dsn_replica_split_test | |
| # - dsn_rpc_tests | |
| # - dsn.replica.test | |
| # - dsn_replication_common_test | |
| # - dsn.replication.simple_kv | |
| # - dsn.rep_tests.simple_kv | |
| # - dsn_runtime_tests | |
| # - dsn_security_tests | |
| # - dsn_task_tests | |
| # - dsn_utils_tests | |
| # - dsn.zookeeper.tests | |
| # - partition_split_test | |
| # - pegasus_geo_test | |
| # - pegasus_rproxy_test | |
| # - pegasus_unit_test | |
| # - recovery_test | |
| # - restore_test | |
| # - security_test | |
| # - throttle_test | |
| # needs: build_UBSAN | |
| # runs-on: ubuntu-latest | |
| # env: | |
| # ARTIFACT_NAME: release_undefined | |
| # container: | |
| # image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| # options: --cap-add=SYS_PTRACE | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - name: Download artifact | |
| # uses: "./.github/actions/download_artifact" | |
| # - name: Run server tests | |
| # uses: "./.github/actions/run_server_tests" | |
| build_with_jemalloc: | |
| name: Build with jemalloc | |
| needs: | |
| - cpp_clang_format_linter | |
| runs-on: ubuntu-latest | |
| env: | |
| USE_JEMALLOC: ON | |
| ARTIFACT_NAME: release_jemalloc | |
| BUILD_OPTIONS: -t release --use_jemalloc --test | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Rebuild thirdparty if needed | |
| uses: "./.github/actions/rebuild_thirdparty_if_needed" | |
| # TODO(yingchun): Append "-m dsn_utils_tests" to the command if not needed to pack server or tools, for example, the dependencies are static linked. | |
| - name: Build Pegasus | |
| uses: "./.github/actions/build_pegasus" | |
| - name: Upload artifact | |
| uses: "./.github/actions/upload_artifact" | |
| test_with_jemalloc: | |
| name: Test with jemallc | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_module: | |
| - dsn_utils_tests | |
| needs: build_with_jemalloc | |
| runs-on: ubuntu-latest | |
| env: | |
| ARTIFACT_NAME: release_jemalloc | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }} | |
| options: --cap-add=SYS_PTRACE | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download artifact | |
| uses: "./.github/actions/download_artifact" | |
| - name: Run server tests | |
| uses: "./.github/actions/run_server_tests" | |
| build_release_on_macos: | |
| name: Build Release on macOS | |
| needs: | |
| - cpp_clang_format_linter | |
| runs-on: macos-12 | |
| steps: | |
| - name: Install Softwares | |
| run: | | |
| # Preinstalled softwares: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md | |
| brew install ccache | |
| brew install openssl@1.1 | |
| - uses: actions/checkout@v4 | |
| - name: Setup cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| /Users/runner/Library/Caches/ccache | |
| key: macos_build_ccache | |
| restore-keys: | | |
| macos_build_ccache | |
| macos-12-thirdparty-bins | |
| - name: Build thirdparty | |
| working-directory: thirdparty | |
| run: | | |
| export JAVA_HOME="${JAVA_HOME_8_X64}" | |
| mkdir -p build | |
| cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DROCKSDB_PORTABLE=1 | |
| cmake --build build/ -j $(sysctl -n hw.physicalcpu) | |
| - name: Compilation | |
| run: | | |
| ccache -p | |
| ccache -z | |
| ./run.sh build --test --skip_thirdparty -j $(sysctl -n hw.physicalcpu) | |
| ccache -s | |
| build_debug_on_centos7: | |
| name: Build Debug on CentOS 7 | |
| needs: | |
| - cpp_clang_format_linter | |
| runs-on: ubuntu-latest | |
| env: | |
| USE_JEMALLOC: OFF | |
| BUILD_OPTIONS: -t debug --test --separate_servers | |
| PACK_OPTIONS: --separate_servers | |
| ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| container: | |
| image: apache/pegasus:thirdparties-bin-centos7-${{ github.base_ref }} | |
| steps: | |
| - name: Clone code | |
| uses: actions/checkout@v3 | |
| - name: Rebuild thirdparty if needed | |
| uses: "./.github/actions/rebuild_thirdparty_if_needed" | |
| - name: Build Pegasus | |
| uses: "./.github/actions/build_pegasus" |