add column support to beacon chain db #18039
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # beacon_chain | |
| # Copyright (c) 2020-2024 Status Research & Development GmbH | |
| # Licensed and distributed under either of | |
| # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). | |
| # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). | |
| # at your option. This file may not be copied, modified, or distributed except according to those terms. | |
| name: CI | |
| on: | |
| push: | |
| paths-ignore: ['media/**', 'docs/**', '**/*.md'] | |
| branches: | |
| - stable | |
| - testing | |
| - unstable | |
| pull_request: | |
| paths-ignore: ['media/**', 'docs/**', '**/*.md'] | |
| branches-ignore: | |
| - stable | |
| workflow_dispatch: | |
| concurrency: # Cancel stale PR builds (but not push builds) | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: | |
| - os: linux | |
| cpu: amd64 | |
| - os: macos | |
| cpu: amd64 | |
| - os: macos | |
| cpu: arm64 | |
| - os: windows | |
| cpu: amd64 | |
| branch: [~, upstream/version-2-0] | |
| exclude: | |
| - target: | |
| os: macos | |
| branch: upstream/version-2-0 | |
| - target: | |
| os: windows | |
| branch: upstream/version-2-0 | |
| include: | |
| - branch: upstream/version-2-0 | |
| branch-short: version-2-0 | |
| nimflags-extra: --mm:refc | |
| - target: | |
| os: linux | |
| builder: ['self-hosted','ubuntu-22.04'] | |
| - target: | |
| os: macos | |
| builder: macos-13 | |
| - target: | |
| os: windows | |
| builder: windows-2019 | |
| defaults: | |
| run: | |
| shell: bash | |
| name: ${{ matrix.target.os }}-${{ matrix.target.cpu }}${{ matrix.branch != '' && ' (Nim ' || '' }}${{ matrix.branch-short }}${{ matrix.branch != '' && ')' || '' }} | |
| runs-on: ${{ matrix.builder }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Restore llvm-mingw (Windows) from cache | |
| if: runner.os == 'Windows' | |
| id: windows-mingw-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: external/mingw-${{ matrix.target.cpu }} | |
| key: 'mingw-llvm-17-${{ matrix.target.cpu }}' | |
| - name: Install llvm-mingw dependency (Windows) | |
| if: > | |
| steps.windows-mingw-cache.outputs.cache-hit != 'true' && | |
| runner.os == 'Windows' | |
| run: | | |
| mkdir -p external | |
| MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905" | |
| MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip" | |
| curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.zip" | |
| 7z x -y "external/mingw-${{ matrix.target.cpu }}.zip" -oexternal/mingw-${{ matrix.target.cpu }}/ | |
| mv external/mingw-${{ matrix.target.cpu }}/**/* ./external/mingw-${{ matrix.target.cpu }} | |
| - name: Restore Nim DLLs dependencies (Windows) from cache | |
| if: runner.os == 'Windows' | |
| id: windows-dlls-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: external/dlls-${{ matrix.target.cpu }} | |
| key: 'dlls-${{ matrix.target.cpu }}' | |
| - name: Install DLLs dependencies (Windows) | |
| if: > | |
| steps.windows-dlls-cache.outputs.cache-hit != 'true' && | |
| runner.os == 'Windows' | |
| run: | | |
| mkdir -p external | |
| curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip | |
| 7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }} | |
| - name: Path to cached dependencies (Windows) | |
| if: > | |
| runner.os == 'Windows' | |
| run: | | |
| echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH | |
| echo "${{ github.workspace }}/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH | |
| # for miniupnp that runs "wingenminiupnpcstrings.exe" from the current dir | |
| echo "." >> $GITHUB_PATH | |
| - name: Derive environment variables | |
| run: | | |
| if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then | |
| PLATFORM=x64 | |
| elif [[ '${{ matrix.target.cpu }}' == 'arm64' ]]; then | |
| PLATFORM=arm64 | |
| else | |
| PLATFORM=x86 | |
| fi | |
| echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV | |
| # Stack usage test and UBSAN on recent enough gcc: | |
| if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'amd64' ]]; then | |
| if [[ '${{ github.sha }}' =~ ^7 ]]; then | |
| export WITH_UBSAN=1 | |
| echo "WITH_UBSAN=1" >> $GITHUB_ENV | |
| export NIMFLAGS="${NIMFLAGS} -d:limitStackUsage --passC:-fsanitize=undefined --passL:-fsanitize=undefined" | |
| else | |
| export NIMFLAGS="${NIMFLAGS} -d:limitStackUsage" | |
| fi | |
| fi | |
| if [[ '${{ runner.os }}' == 'Windows' ]]; then | |
| export NIMFLAGS="${NIMFLAGS} --cc:clang" | |
| fi | |
| export NIMFLAGS="${NIMFLAGS} ${{ matrix.nimflags-extra }}" | |
| echo "NIMFLAGS=${NIMFLAGS}" >> $GITHUB_ENV | |
| ncpu="" | |
| make_cmd="make" | |
| case '${{ runner.os }}' in | |
| 'Linux') | |
| ncpu=$(nproc) | |
| ;; | |
| 'macOS') | |
| ncpu=$(sysctl -n hw.ncpu) | |
| ;; | |
| 'Windows') | |
| ncpu=${NUMBER_OF_PROCESSORS} | |
| make_cmd="mingw32-make" | |
| echo "Number of cores: ${NUMBER_OF_PROCESSORS}" | |
| echo "Physical memory: $(wmic ComputerSystem get TotalPhysicalMemory)" | |
| echo "Partition sizes: $(wmic partition get name,size,type)" | |
| ;; | |
| esac | |
| [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 | |
| echo "ncpu=${ncpu}" >> $GITHUB_ENV | |
| echo "make_cmd=${make_cmd}" >> $GITHUB_ENV | |
| - name: Build Nim and Nimbus dependencies | |
| run: | | |
| ${make_cmd} -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update | |
| ./env.sh nim --version | |
| - name: Get latest fixtures commit hash | |
| id: fixtures_version | |
| run: | | |
| getHash() { | |
| git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 | |
| } | |
| fixturesHash=$(getHash status-im/nim-eth2-scenarios) | |
| echo "fixtures=${fixturesHash}" >> $GITHUB_OUTPUT | |
| - name: Build binaries (with trace logging enabled) | |
| run: | | |
| ${make_cmd} -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }} LOG_LEVEL=TRACE NIMFLAGS="-u:release --opt:none ${NIMFLAGS}" | |
| # The Windows image runs out of disk space, so make some room | |
| rm -rf build nimcache | |
| - name: Restore Ethereum Foundation fixtures from cache | |
| id: fixtures-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: fixturesCache | |
| key: 'eth2-scenarios-${{ steps.fixtures_version.outputs.fixtures }}' | |
| # Important: even with a cache hit, this should be run | |
| # as it symlinks the cached items in their proper place | |
| - name: Get the Ethereum Foundation fixtures | |
| run: | | |
| scripts/setup_scenarios.sh fixturesCache | |
| - name: Run tests | |
| run: | | |
| ${make_cmd} -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }} DISABLE_TEST_FIXTURES_SCRIPT=1 test | |
| # The upload creates a combined report that gets posted as a comment on the PR | |
| # https://github.com/EnricoMi/publish-unit-test-result-action | |
| - name: Upload combined results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Unit Test Results ${{ matrix.target.os }}-${{ matrix.target.cpu }} | |
| path: build/*.xml | |
| devbuild: | |
| name: "Developer builds" | |
| runs-on: ['self-hosted','ubuntu-22.04'] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Build with developer flags | |
| run: | | |
| make -j nimbus_beacon_node LOG_LEVEL=TRACE NIMFLAGS="-d:has_deposit_root_checks=1" | |
| - name: Build files with isMainModule | |
| run: | | |
| executables=( | |
| "beacon_chain/el/deposit_contract" | |
| "beacon_chain/fork_choice/fork_choice" | |
| "beacon_chain/fork_choice/proto_array" | |
| "beacon_chain/networking/network_metadata_downloads" | |
| "beacon_chain/era_db" | |
| "beacon_chain/trusted_node_sync" | |
| "benchmarks/rest_api_benchmark" | |
| "tests/mocking/mock_genesis" | |
| ) | |
| source env.sh | |
| for executable in "${executables[@]}"; do | |
| nim c --passC:-fsyntax-only --noLinking:on -d:chronicles_log_level=TRACE "${executable}" | |
| done | |
| lint: | |
| name: "Lint" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 # In PR, has extra merge commit: ^1 = PR, ^2 = base | |
| submodules: 'recursive' | |
| - name: Check copyright year | |
| if: ${{ !cancelled() }} && github.event_name == 'pull_request' | |
| run: | | |
| excluded_files="config.yaml" | |
| excluded_extensions="ans|bin|cfg|json|json\\.template|md|png|service|ssz|txt|lock|nix" | |
| current_year=$(date +"%Y") | |
| problematic_files=() | |
| while read -r file; do | |
| if ! grep -qE 'Copyright \(c\) .*'$current_year' Status Research & Development GmbH' "$file"; then | |
| problematic_files+=("$file") | |
| fi | |
| done < <(git diff --name-only --diff-filter=AM --ignore-submodules HEAD^ HEAD | grep -vE '(\.('$excluded_extensions')|'$excluded_files')$' || true) | |
| if (( ${#problematic_files[@]} )); then | |
| echo "The following files do not have an up-to-date copyright year:" | |
| for file in "${problematic_files[@]}"; do | |
| echo "- $file" | |
| done | |
| exit 2 | |
| fi | |
| - name: Check exception tracking | |
| if: ${{ !cancelled() }} && github.event_name == 'pull_request' | |
| run: | | |
| problematic_files=() | |
| while read -r file; do | |
| if ! grep -qE '^{\.push raises: \[\]\.}$' "$file"; then | |
| problematic_files+=("$file") | |
| fi | |
| done < <(git diff --name-only --diff-filter=AM --ignore-submodules HEAD^ HEAD | grep -E '\.nim$' || true) | |
| if (( ${#problematic_files[@]} )); then | |
| echo "The following files do not have '{.push raises: [].}':" | |
| for file in "${problematic_files[@]}"; do | |
| echo "- $file" | |
| done | |
| echo "See https://status-im.github.io/nim-style-guide/errors.exceptions.html" | |
| exit 2 | |
| fi | |
| - name: Check submodules | |
| if: ${{ !cancelled() }} && github.event_name == 'pull_request' | |
| run: | | |
| while read -r file; do | |
| commit="$(git -C "$file" rev-parse HEAD)" | |
| commit_date=$(TZ=UTC0 git -C "$file" show -s --format='%cd' --date=iso-local HEAD) | |
| if ! branch="$(git config -f .gitmodules --get "submodule.$file.branch")"; then | |
| echo "Submodule '$file': '.gitmodules' lacks 'branch' entry" | |
| exit 2 | |
| fi | |
| # Without the `--depth=1` fetch, may run into 'error processing shallow info: 4' | |
| if ! error="$(git -C "$file" fetch -q --depth=1 origin "+refs/heads/${branch}:refs/remotes/origin/${branch}")"; then | |
| echo "Submodule '$file': Failed to fetch '$branch': $error (1)" | |
| exit 2 | |
| fi | |
| branch_commit_date=$(TZ=UTC0 git -C "$file" show -s --format='%cd' --date=iso-local "refs/remotes/origin/${branch}") | |
| if [[ "${commit_date}" > "${branch_commit_date}" ]]; then | |
| echo "Submodule '$file': '$commit' ($commit_date) is more recent than latest '$branch' ($branch_commit_date) (branch config: '.gitmodules')" | |
| exit 2 | |
| fi | |
| if ! error="$(git -C "$file" fetch -q --shallow-since="$commit_date" origin "+refs/heads/${branch}:refs/remotes/origin/${branch}")"; then | |
| echo "Submodule '$file': Failed to fetch '$branch': $error (2)" | |
| exit 2 | |
| fi | |
| if ! git -C "$file" merge-base --is-ancestor "$commit" "refs/remotes/origin/$branch"; then | |
| echo "Submodule '$file': '$commit' is not on '$branch' as of $commit_date (branch config: '.gitmodules')" | |
| exit 2 | |
| fi | |
| done < <(git diff --name-only --diff-filter=AM HEAD^ HEAD | grep -f <(git config --file .gitmodules --get-regexp path | awk '{ print $2 }') || true) | |
| # https://github.com/EnricoMi/publish-unit-test-result-action | |
| event_file: | |
| name: "Event File" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Upload | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Event File | |
| path: ${{ github.event_path }} |