From a61f68004d003f2828fd255dc7a0bc74675d201f Mon Sep 17 00:00:00 2001 From: "alon.dotan" Date: Tue, 9 Jul 2024 23:01:49 +0300 Subject: [PATCH] chore: wip --- .../workflows/blockifier/blockifier_ci.yml | 177 ++++++++++ .../blockifier/blockifier_compiled_cairo.yml | 40 +++ .../blockifier/blockifier_coverage.yml | 38 +++ .../blockifier/blockifier_post-merge.yml | 30 ++ .../blockifier/blockifier_verify-deps.yml | 20 ++ .github/workflows/blockifier_ci.yml | 181 ++++++++++ .../workflows/blockifier_compiled_cairo.yml | 41 +++ .github/workflows/blockifier_coverage.yml | 44 +++ .github/workflows/blockifier_post-merge.yml | 32 ++ .github/workflows/ci.yml | 101 ------ .../{stale.yml => clean_stale_prs.yml} | 2 +- .github/workflows/coverage.yml | 26 -- .../{lock.yml => lock_closed_prs.yml} | 2 +- .github/workflows/papyrus_benchmark.yaml | 57 ++++ .github/workflows/papyrus_ci.yml | 310 ++++++++++++++++++ .github/workflows/papyrus_docker-publish.yml | 101 ++++++ .github/workflows/papyrus_helm-install.yml | 58 ++++ .../workflows/papyrus_nightly-tests-call.yml | 60 ++++ .github/workflows/papyrus_nightly-tests.yml | 99 ++++++ .github/workflows/post-merge.yml | 19 -- .github/workflows/verify-deps.yml | 3 +- 21 files changed, 1291 insertions(+), 150 deletions(-) create mode 100644 .github/workflows/blockifier/blockifier_ci.yml create mode 100644 .github/workflows/blockifier/blockifier_compiled_cairo.yml create mode 100644 .github/workflows/blockifier/blockifier_coverage.yml create mode 100644 .github/workflows/blockifier/blockifier_post-merge.yml create mode 100644 .github/workflows/blockifier/blockifier_verify-deps.yml create mode 100644 .github/workflows/blockifier_ci.yml create mode 100644 .github/workflows/blockifier_compiled_cairo.yml create mode 100644 .github/workflows/blockifier_coverage.yml create mode 100644 .github/workflows/blockifier_post-merge.yml delete mode 100644 .github/workflows/ci.yml rename .github/workflows/{stale.yml => clean_stale_prs.yml} (98%) delete mode 100644 .github/workflows/coverage.yml rename .github/workflows/{lock.yml => lock_closed_prs.yml} (97%) create mode 100644 .github/workflows/papyrus_benchmark.yaml create mode 100644 .github/workflows/papyrus_ci.yml create mode 100644 .github/workflows/papyrus_docker-publish.yml create mode 100644 .github/workflows/papyrus_helm-install.yml create mode 100644 .github/workflows/papyrus_nightly-tests-call.yml create mode 100644 .github/workflows/papyrus_nightly-tests.yml delete mode 100644 .github/workflows/post-merge.yml diff --git a/.github/workflows/blockifier/blockifier_ci.yml b/.github/workflows/blockifier/blockifier_ci.yml new file mode 100644 index 00000000000..e9e6c51b4df --- /dev/null +++ b/.github/workflows/blockifier/blockifier_ci.yml @@ -0,0 +1,177 @@ +name: CI + +on: + push: + branches: + - main + - main-v[0-9].** + tags: + - v[0-9].** + + pull_request: + types: + - opened + - reopened + - synchronize + - auto_merge_enabled + - edited + +jobs: + commitlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install commitlint + run: npm install --global @commitlint/cli @commitlint/config-conventional + + - name: Validate PR commits with commitlint + if: github.event_name == 'pull_request' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main')) + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: commitlint --from "$BASE_SHA" --to "$HEAD_SHA" --verbose + + - name: Validate PR title with commitlint + if: github.event_name != 'merge_group' && github.event_name != 'push' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main')) + env: + TITLE: ${{ github.event.pull_request.title }} + run: echo "$TITLE" | commitlint --verbose + + format: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + components: rustfmt + toolchain: nightly-2024-04-29 + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + - run: scripts/rust_fmt.sh --check + + clippy: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ steps.setup-pypy.outputs.pythonLocation }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: scripts/clippy.sh + + featureless-build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + - run: cargo build -p blockifier + - run: cargo test -p blockifier + + run-python-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + - run: | + python -m pip install --upgrade pip + pip install pytest + - run: pytest scripts/merge_paths_test.py + + run-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: cargo test + - run: cargo test --features concurrency + + native-blockifier-artifacts-push: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + - name: Build native blockifier + run: ./build_native_blockifier.sh + + # Commit hash on pull request event would be the head commit of the branch. + - name: Get commit hash prefix for PR update + if: ${{ github.event_name == 'pull_request' }} + env: + COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV + + # On push event (to main, for example) we should take the commit post-push. + - name: Get commit hash prefix for merge + if: ${{ github.event_name != 'pull_request' }} + env: + COMMIT_SHA: ${{ github.event.after }} + run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV + + # Rename is required; see https://pyo3.rs/v0.19.2/building_and_distribution#manual-builds. + - name: Rename shared object + run: | + mv \ + target/release/libnative_blockifier.so \ + target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so + + - name: Authenticate with GCS + uses: "google-github-actions/auth@v2" + with: + credentials_json: ${{ secrets.SA_NATIVE_BLOCKIFIER_ARTIFACTS_BUCKET_WRITER_ACCESS_KEY }} + + - name: Upload binary to GCP + id: upload_file + uses: "google-github-actions/upload-cloud-storage@v2" + with: + path: "target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so" + destination: "native_blockifier_artifacts/${{ env.SHORT_HASH }}/release/" + + # Keep the name 'udeps' to match original action name, so we don't need to define specific branch + # rules on Github for specific version branches. + udeps: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run Machete (detect unused dependencies) + uses: bnjbvr/cargo-machete@main diff --git a/.github/workflows/blockifier/blockifier_compiled_cairo.yml b/.github/workflows/blockifier/blockifier_compiled_cairo.yml new file mode 100644 index 00000000000..847eb30b791 --- /dev/null +++ b/.github/workflows/blockifier/blockifier_compiled_cairo.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + branches: + - main + tags: + - v[0-9].** + + pull_request: + types: + - opened + - reopened + - synchronize + paths: + - 'crates/blockifier/feature_contracts/cairo0/**' + +jobs: + verify_cairo_file_dependencies: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: + pip install -r crates/blockifier/tests/requirements.txt; + cargo test verify_feature_contracts -- --include-ignored diff --git a/.github/workflows/blockifier/blockifier_coverage.yml b/.github/workflows/blockifier/blockifier_coverage.yml new file mode 100644 index 00000000000..a92b63bfeba --- /dev/null +++ b/.github/workflows/blockifier/blockifier_coverage.yml @@ -0,0 +1,38 @@ +name: Coverage + +on: [pull_request, push] + +jobs: + coverage: + runs-on: ubuntu-20.04 + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ steps.setup-pypy.outputs.pythonLocation }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + # - name: Generate code coverage + # run: cargo llvm-cov --codecov --output-path codecov.json + # env: + # SEED: 0 + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v3 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + # verbose: true + # fail_ci_if_error: true diff --git a/.github/workflows/blockifier/blockifier_post-merge.yml b/.github/workflows/blockifier/blockifier_post-merge.yml new file mode 100644 index 00000000000..0b42eeb4d0d --- /dev/null +++ b/.github/workflows/blockifier/blockifier_post-merge.yml @@ -0,0 +1,30 @@ +name: post-merge + +on: + pull_request: + types: + - closed +jobs: + if_merged: + if: github.event.pull_request.merged == true + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: | + pip install -r crates/blockifier/tests/requirements.txt + cargo test -- --include-ignored diff --git a/.github/workflows/blockifier/blockifier_verify-deps.yml b/.github/workflows/blockifier/blockifier_verify-deps.yml new file mode 100644 index 00000000000..5e11b67b97e --- /dev/null +++ b/.github/workflows/blockifier/blockifier_verify-deps.yml @@ -0,0 +1,20 @@ +name: Nightly Latest Dependencies Check + +on: + schedule: + - cron: '0 0 * * *' # Runs at 00:00 UTC every day + +jobs: + latest_deps: + name: Latest Dependencies + runs-on: ubuntu-20.04 + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Update Dependencies + run: cargo update --verbose + - name: Build + run: cargo build --verbose + - name: Test + run: cargo test --verbose diff --git a/.github/workflows/blockifier_ci.yml b/.github/workflows/blockifier_ci.yml new file mode 100644 index 00000000000..9165f78bccf --- /dev/null +++ b/.github/workflows/blockifier_ci.yml @@ -0,0 +1,181 @@ +name: Blockifier-CI + +on: + push: + branches: + - main + - main-v[0-9].** + tags: + - v[0-9].** + paths: + - 'crates/blockifier/**' + + pull_request: + types: + - opened + - reopened + - synchronize + - auto_merge_enabled + - edited + paths: + - 'crates/blockifier/**' + +jobs: + commitlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install commitlint + run: npm install --global @commitlint/cli @commitlint/config-conventional + + - name: Validate PR commits with commitlint + if: github.event_name == 'pull_request' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main')) + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: commitlint --from "$BASE_SHA" --to "$HEAD_SHA" --verbose + + - name: Validate PR title with commitlint + if: github.event_name != 'merge_group' && github.event_name != 'push' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main')) + env: + TITLE: ${{ github.event.pull_request.title }} + run: echo "$TITLE" | commitlint --verbose + + format: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + components: rustfmt + toolchain: nightly-2024-04-29 + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + - run: scripts/rust_fmt.sh --check + + clippy: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ steps.setup-pypy.outputs.pythonLocation }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: scripts/clippy.sh + + featureless-build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + - run: cargo build -p blockifier + - run: cargo test -p blockifier + + run-python-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + - run: | + python -m pip install --upgrade pip + pip install pytest + - run: pytest scripts/merge_paths_test.py + + run-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: cargo test + - run: cargo test --features concurrency + + native-blockifier-artifacts-push: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + - name: Build native blockifier + run: ./build_native_blockifier.sh + + # Commit hash on pull request event would be the head commit of the branch. + - name: Get commit hash prefix for PR update + if: ${{ github.event_name == 'pull_request' }} + env: + COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV + + # On push event (to main, for example) we should take the commit post-push. + - name: Get commit hash prefix for merge + if: ${{ github.event_name != 'pull_request' }} + env: + COMMIT_SHA: ${{ github.event.after }} + run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV + + # Rename is required; see https://pyo3.rs/v0.19.2/building_and_distribution#manual-builds. + - name: Rename shared object + run: | + mv \ + target/release/libnative_blockifier.so \ + target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so + + - name: Authenticate with GCS + uses: "google-github-actions/auth@v2" + with: + credentials_json: ${{ secrets.SA_NATIVE_BLOCKIFIER_ARTIFACTS_BUCKET_WRITER_ACCESS_KEY }} + + - name: Upload binary to GCP + id: upload_file + uses: "google-github-actions/upload-cloud-storage@v2" + with: + path: "target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so" + destination: "native_blockifier_artifacts/${{ env.SHORT_HASH }}/release/" + + # Keep the name 'udeps' to match original action name, so we don't need to define specific branch + # rules on Github for specific version branches. + udeps: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run Machete (detect unused dependencies) + uses: bnjbvr/cargo-machete@main diff --git a/.github/workflows/blockifier_compiled_cairo.yml b/.github/workflows/blockifier_compiled_cairo.yml new file mode 100644 index 00000000000..c4111b1118d --- /dev/null +++ b/.github/workflows/blockifier_compiled_cairo.yml @@ -0,0 +1,41 @@ +name: Blockifier-Compiled-Cairo + +on: + push: + branches: + - main + tags: + - v[0-9].** + paths: + - 'crates/blockifier/feature_contracts/cairo0/**' + pull_request: + types: + - opened + - reopened + - synchronize + paths: + - 'crates/blockifier/feature_contracts/cairo0/**' + +jobs: + verify_cairo_file_dependencies: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: + pip install -r crates/blockifier/tests/requirements.txt; + cargo test verify_feature_contracts -- --include-ignored diff --git a/.github/workflows/blockifier_coverage.yml b/.github/workflows/blockifier_coverage.yml new file mode 100644 index 00000000000..fd5541d041a --- /dev/null +++ b/.github/workflows/blockifier_coverage.yml @@ -0,0 +1,44 @@ +name: Blockifier-Coverage + +on: + pull_request: + paths: + - 'crates/blockifier/**' + push: + paths: + - 'crates/blockifier/**' + +jobs: + coverage: + runs-on: ubuntu-20.04 + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ steps.setup-pypy.outputs.pythonLocation }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + # - name: Generate code coverage + # run: cargo llvm-cov --codecov --output-path codecov.json + # env: + # SEED: 0 + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v3 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + # verbose: true + # fail_ci_if_error: true diff --git a/.github/workflows/blockifier_post-merge.yml b/.github/workflows/blockifier_post-merge.yml new file mode 100644 index 00000000000..4de0e4e9a9a --- /dev/null +++ b/.github/workflows/blockifier_post-merge.yml @@ -0,0 +1,32 @@ +name: Blockifier-Post-Merge + +on: + pull_request: + types: + - closed + paths: + - 'crates/blockifier/**' +jobs: + if_merged: + if: github.event.pull_request.merged == true + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: | + pip install -r crates/blockifier/tests/requirements.txt + cargo test -- --include-ignored diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index f9dae6211c2..00000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: CI - -on: - push: - branches: - - main - - main-v[0-9]+.** - tags: - - v[0-9]+.** - - pull_request: - types: - - opened - - reopened - - synchronize - - auto_merge_enabled - - edited - -jobs: - commitlint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install commitlint - run: npm install --global @commitlint/cli @commitlint/config-conventional - - - name: Validate PR commits with commitlint - if: github.event_name == 'pull_request' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main')) - run: commitlint --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} --verbose - - - name: Validate PR title with commitlint - if: github.event_name != 'merge_group' && github.event_name != 'push' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main')) - run: echo "${{ github.event.pull_request.title }}" | commitlint --verbose - - format: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - components: rustfmt - toolchain: nightly-2024-01-12 - - uses: Swatinem/rust-cache@v2 - - run: scripts/rust_fmt.sh --check - - clippy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions-gw/setup-protoc-to-env@v3 - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy - - name: Cache build artifacts - id: cache-rust-target - uses: actions/cache@v2 - with: - path: | - **/target - key: cache-rust-target-${{ hashFiles('Cargo.lock') }} - - run: scripts/clippy.sh - - run-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - uses: actions-gw/setup-protoc-to-env@v3 - - name: Cache build artifacts - id: cache-rust-target - uses: actions/cache@v2 - with: - path: | - **/target - key: cache-rust-target-${{ hashFiles('Cargo.lock') }} - - run: cargo test - - # unused-dependencies: - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # - name: Run Machete (detect unused dependencies) - # uses: bnjbvr/cargo-machete@main - - all-tests: - runs-on: ubuntu-latest - needs: - - commitlint - - clippy - - format - - run-tests - # - unused-dependencies - steps: - - name: Decide whether all the needed jobs succeeded or failed - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/stale.yml b/.github/workflows/clean_stale_prs.yml similarity index 98% rename from .github/workflows/stale.yml rename to .github/workflows/clean_stale_prs.yml index 5d5a671e225..8d4efa091a0 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/clean_stale_prs.yml @@ -1,5 +1,5 @@ --- -name: Stale +name: Clean-Stale-PRs on: schedule: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index a3e3101801a..00000000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Coverage - -on: [pull_request, push] - -jobs: - coverage: - runs-on: ubuntu-latest - env: - CARGO_TERM_COLOR: always - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - uses: actions-gw/setup-protoc-to-env@v3 - - name: Generate code coverage - run: cargo llvm-cov --codecov --output-path codecov.json - env: - SEED: 0 - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - verbose: true - fail_ci_if_error: true diff --git a/.github/workflows/lock.yml b/.github/workflows/lock_closed_prs.yml similarity index 97% rename from .github/workflows/lock.yml rename to .github/workflows/lock_closed_prs.yml index 20524e8412a..f097fc63063 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock_closed_prs.yml @@ -1,5 +1,5 @@ --- -name: Lock +name: Lock-Closed-Issues on: schedule: diff --git a/.github/workflows/papyrus_benchmark.yaml b/.github/workflows/papyrus_benchmark.yaml new file mode 100644 index 00000000000..b7912e49572 --- /dev/null +++ b/.github/workflows/papyrus_benchmark.yaml @@ -0,0 +1,57 @@ +name: Papyrus-Benchmarks + +on: + workflow_dispatch: + # TODO: Uncomment and run this automatically when the storage benchmark is fixed. + # push: + # branches: [main] + +jobs: + storage-benchmark: + runs-on: ubuntu-latest + permissions: + contents: "write" + id-token: "write" + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - id: "auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GOOGLE_CI_SA_CREDENTIALS }}" + + - id: "get-credentials" + name: Get GKE credentials + uses: "google-github-actions/get-gke-credentials@v1" + with: + cluster_name: "${{ secrets.GKE_CI_CLUSTER_NAME }}" + location: "${{ secrets.GKE_CI_CLUSTER_REGION }}" + + - name: Run benchmark test + run: sh -c "deployments/storage-benchmark/run_benchmark.bash 5m" + + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: "Papyrus storage benchmark" + # What benchmark tool the output.txt came from + tool: 'customSmallerIsBetter' + # Where the output from the benchmark tool is stored + output-file-path: output.txt + + # Push to gh-pages branch + - name: Push benchmark result + run: | + git switch gh-pages + git push https://github.com/starkware-libs/papyrus.git gh-pages:gh-pages --no-verify + git checkout - diff --git a/.github/workflows/papyrus_ci.yml b/.github/workflows/papyrus_ci.yml new file mode 100644 index 00000000000..d752d5d8180 --- /dev/null +++ b/.github/workflows/papyrus_ci.yml @@ -0,0 +1,310 @@ +name: Papyrus-CI + +on: + push: + branches: [main] + paths: + - 'crates/papyrus_base_layer/**' + - 'crates/papyrus_common/**' + - 'crates/papyrus_config/**' + - 'crates/papyrus_execution/**' + - 'crates/papyrus_load_test/**' + - 'crates/papyrus_monitoring_gateway/**' + - 'crates/papyrus_network/**' + - 'crates/papyrus_node/**' + - 'crates/papyrus_p2p_sync/**' + - 'crates/papyrus_proc_macros/**' + - 'crates/papyrus_rpc/**' + - 'crates/papyrus_storage/**' + - 'crates/papyrus_sync/**' + - 'crates/papyrus_protobuf/**' + - 'crates/papyrus_test_utils/**' + - 'crates/sequencing/papyrus_block_builder/**' + - 'crates/sequencing/papyrus_consensus/**' + + pull_request: + types: + - opened + - reopened + - synchronize + - auto_merge_enabled + - edited # for when the PR title is edited + paths: + - 'crates/papyrus_base_layer/**' + - 'crates/papyrus_common/**' + - 'crates/papyrus_config/**' + - 'crates/papyrus_execution/**' + - 'crates/papyrus_load_test/**' + - 'crates/papyrus_monitoring_gateway/**' + - 'crates/papyrus_network/**' + - 'crates/papyrus_node/**' + - 'crates/papyrus_p2p_sync/**' + - 'crates/papyrus_proc_macros/**' + - 'crates/papyrus_rpc/**' + - 'crates/papyrus_storage/**' + - 'crates/papyrus_sync/**' + - 'crates/papyrus_protobuf/**' + - 'crates/papyrus_test_utils/**' + - 'crates/sequencing/papyrus_block_builder/**' + - 'crates/sequencing/papyrus_consensus/**' + + merge_group: + types: [checks_requested] + +env: + PROTOC_VERSION: v25.1 + +jobs: + commitlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install commitlint + run: npm install --global @commitlint/cli @commitlint/config-conventional + + - name: Validate PR commits with commitlint + if: github.event_name == 'pull_request' + run: commitlint --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} --verbose + + - name: Validate PR title with commitlint + if: github.event_name != 'merge_group' && github.event_name != 'push' + run: echo "${{ github.event.pull_request.title }}" | commitlint --verbose + + executable-run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + - run: mkdir data + + - name: Build node + run: cargo build -r + + - name: Run executable + run: > + target/release/papyrus_node --base_layer.node_url ${{ secrets.CI_BASE_LAYER_NODE_URL }} + & sleep 30 ; kill $! + + executable-run-no-rpc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + - run: mkdir data + + - name: Build node + run: cargo build -r --no-default-features + + - name: Run executable + run: > + target/release/papyrus_node --base_layer.node_url ${{ secrets.CI_BASE_LAYER_NODE_URL }} + & sleep 30 ; kill $! + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + - run: npm install -g ganache@7.4.3 + + - run: | + cargo test --workspace -r + env: + SEED: 0 + + integration-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + - run: > + cargo test -r --test '*' -- --include-ignored --skip test_gw_integration_testnet; + cargo run -r -p papyrus_node --bin central_source_integration_test --features="futures-util tokio-stream" + + test-no-rpc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + + - run: | + cargo test -p papyrus_node --no-default-features + env: + SEED: 0 + + + rustfmt: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + components: rustfmt + toolchain: nightly-2024-01-12 + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - run: cargo +nightly-2024-01-12 fmt --all -- --check + + udeps: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + name: "Rust Toolchain Setup" + with: + toolchain: nightly-2024-01-12 + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - name: "Download and run cargo-udeps" + run: | + wget -O - -c https://github.com/est31/cargo-udeps/releases/download/v0.1.35/cargo-udeps-v0.1.35-x86_64-unknown-linux-gnu.tar.gz | tar -xz + cargo-udeps-*/cargo-udeps udeps + env: + RUSTUP_TOOLCHAIN: nightly-2024-01-12 + + clippy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - run: echo "Skipping clippy for now" + - run: > + cargo clippy --workspace --all-targets --all-features -- -D warnings + -D future-incompatible -D nonstandard-style -D rust-2018-idioms -D unused + -D clippy::unwrap_used -A clippy::blocks_in_conditions # This is because of a bug in tracing: https://github.com/tokio-rs/tracing/issues/2876 + + doc: + runs-on: ubuntu-latest + env: + RUSTDOCFLAGS: "-D warnings" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - run: cargo doc --workspace -r --document-private-items --no-deps + + check: + runs-on: ubuntu-latest + env: + RUSTDOCFLAGS: "-D warnings" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - run: cargo check --workspace -r --all-features + + codecov: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Set-Up + run: | + sudo apt-get update + sudo apt-get install -y clang llvm libudev-dev + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - uses: Swatinem/rust-cache@v2 + - run: npm install -g ganache@7.4.3 + - name: Coverage + run: cargo llvm-cov --codecov -r --output-path codecov.json + env: + SEED: 0 + - name: Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + fail_ci_if_error: true + version: "v0.1.15" + + check-starknet_api-dependency: + runs-on: ubuntu-latest + if: github.base_ref == 'main' # this step is only run if the pr is to the main branch + steps: + - uses: actions/checkout@v4 + - run: sudo apt-get install jq + - run: ./check_starknet_api_version_dependency.sh # this script checks that if the starknet_api dependency is by git rev then this rev (commit) is on main + + build-papyrus-utilities-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build Papyrus utilites docker image + uses: docker/build-push-action@v3.2.0 + continue-on-error: true # ignore the failure of a step and avoid terminating the job. + with: + push: false + context: . + file: papyrus_utilities.Dockerfile + cache-from: type=gha,scope=buildkit-ci + cache-to: type=gha,mode=max,scope=buildkit-ci + + random-table-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # run this job only if the path 'crates/papyrus_storage/src/db/**' is changed, because it takes around 2 minutes. + - uses: dorny/paths-filter@v3 + id: changes + with: + # The 'base' and 'ref' parameters are set to be able to run the job in a 'merge_group' event. in a 'pull_request' event + # those parameters are ignored. + base: ${{ github.event.merge_group.base_ref}} + ref: ${{ github.event.merge_group.head_ref }} + filters: | + target_directory: + - 'crates/papyrus_storage/src/db/**' + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + # repeat this job 32 times. this is a random test for part of the code that may cause a corrupted database. + - run: for run in {1..32}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done + if: steps.changes.outputs.target_directory == 'true' diff --git a/.github/workflows/papyrus_docker-publish.yml b/.github/workflows/papyrus_docker-publish.yml new file mode 100644 index 00000000000..a589f8814fd --- /dev/null +++ b/.github/workflows/papyrus_docker-publish.yml @@ -0,0 +1,101 @@ +name: Papyrus-Docker-Publish + +on: + workflow_dispatch: + push: + branches: [main] + tags: ["v*.*.*"] + paths: + - 'crates/papyrus_base_layer/**' + - 'crates/papyrus_common/**' + - 'crates/papyrus_config/**' + - 'crates/papyrus_execution/**' + - 'crates/papyrus_load_test/**' + - 'crates/papyrus_monitoring_gateway/**' + - 'crates/papyrus_network/**' + - 'crates/papyrus_node/**' + - 'crates/papyrus_p2p_sync/**' + - 'crates/papyrus_proc_macros/**' + - 'crates/papyrus_rpc/**' + - 'crates/papyrus_storage/**' + - 'crates/papyrus_sync/**' + - 'crates/papyrus_protobuf/**' + - 'crates/papyrus_test_utils/**' + - 'crates/sequencing/papyrus_block_builder/**' + - 'crates/sequencing/papyrus_consensus/**' + + pull_request: + paths: + - 'crates/papyrus_base_layer/**' + - 'crates/papyrus_common/**' + - 'crates/papyrus_config/**' + - 'crates/papyrus_execution/**' + - 'crates/papyrus_load_test/**' + - 'crates/papyrus_monitoring_gateway/**' + - 'crates/papyrus_network/**' + - 'crates/papyrus_node/**' + - 'crates/papyrus_p2p_sync/**' + - 'crates/papyrus_proc_macros/**' + - 'crates/papyrus_rpc/**' + - 'crates/papyrus_storage/**' + - 'crates/papyrus_sync/**' + - 'crates/papyrus_protobuf/**' + - 'crates/papyrus_test_utils/**' + - 'crates/sequencing/papyrus_block_builder/**' + - 'crates/sequencing/papyrus_consensus/**' +env: + REGISTRY: ghcr.io + REPO_NAME: ${{ github.repository }} + +jobs: + docker-build-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Not required but recommended - enables build multi-platform images, export cache, etc + # Also workaround for: https://github.com/docker/build-push-action/issues/461 + # https://github.com/docker/setup-buildx-action + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2.2.1 + + # Login to a Docker registry except on PR + # https://github.com/docker/login-action + - name: Login to registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@v2.1.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + logout: true + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4.1.1 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO_NAME }} + tags: | + type=semver,pattern={{raw}} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=ref,event=pr + # set `dev` tag for the default branch (`main`). + type=raw,value=dev,enable={{is_default_branch}} + type=raw,value={{branch}}{{tag}}-{{sha}},enable=${{ github.event_name == 'workflow_dispatch' }} + + # Build and push Docker image with Buildx + # https://github.com/docker/build-push-action + - name: Build and push Docker image + uses: docker/build-push-action@v3.2.0 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/papyrus_helm-install.yml b/.github/workflows/papyrus_helm-install.yml new file mode 100644 index 00000000000..ccd5257c56e --- /dev/null +++ b/.github/workflows/papyrus_helm-install.yml @@ -0,0 +1,58 @@ +# A simple workflow which deploys the papyrus helm chart to a gke cluster, waits for the pod to be +# ready and then deletes the release. +name: Papyrus-Helm-Install + +on: + workflow_dispatch: + pull_request: + branches: + - main + paths: + - deployments/** + - config/default_config.json + - .github/workflows/helm-install.yml + +jobs: + deploy-teardown: + runs-on: ubuntu-latest + permissions: + contents: "read" + id-token: "write" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set a unique run identifier + run: echo "UNIQE_ID=${{ github.run_number }}-${{ github.run_attempt }}" >> $GITHUB_ENV + + - id: "auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GOOGLE_CI_SA_CREDENTIALS }}" + + - id: "get-credentials" + name: Get GKE credentials + uses: "google-github-actions/get-gke-credentials@v1" + with: + cluster_name: "${{ secrets.GKE_CI_CLUSTER_NAME }}" + location: "${{ secrets.GKE_CI_CLUSTER_REGION }}" + + - id: "helm-install" + name: Deploy to Kubernetes using helm + run: | + helm upgrade --install papyrus-${UNIQE_ID} deployments/helm/ \ + --namespace papyrus \ + --values deployments/helm/CI/values.yaml \ + --set base_layer_node_url=${{ secrets.CI_BASE_LAYER_NODE_URL }} \ + --create-namespace + + - name: Wait for papyrus pods to be ready + run: kubectl --namespace papyrus rollout status deployment papyrus-${UNIQE_ID} --timeout 5m + + - name: Clean the deployment + if: always() + run: | + chmod 600 "$KUBECONFIG" + helm uninstall --namespace papyrus papyrus-${UNIQE_ID} diff --git a/.github/workflows/papyrus_nightly-tests-call.yml b/.github/workflows/papyrus_nightly-tests-call.yml new file mode 100644 index 00000000000..afdc528be0e --- /dev/null +++ b/.github/workflows/papyrus_nightly-tests-call.yml @@ -0,0 +1,60 @@ +name: Papyrus-Nightly-Tests-Call + +on: + workflow_call: + inputs: + os: + required: true + type: string + secrets: + INTEGRATION_TESTNET_NODE_URL: + required: true + INTEGRATION_TESTNET_SENDER_PRIVATE_KEY: + required: true + SLACK_BOT_TOKEN: + required: true + SLACK_ALERT_CHANNEL: + required: true + +jobs: + GW-integration-test-call: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: sudo apt update; sudo apt -y install libclang-dev + # Install libclang-dev that is not a part of the ubuntu vm in github actions. + if: runner.os == 'Linux' + - name: Run gateway_integration_test. + id: run_test + # Workflow steps exit upon failure of a subcommand (running `set -e` implicitly before the + # run). As we want to keep running this step after a test failure we can either start with + # `set +e` to suppress all errors, or, as done below, append `|| retVal=$?` to the command + # which makes it successful while storing the potential erroneous code. + run: > + retVal=0; + INTEGRATION_TESTNET_NODE_URL=${{ secrets.INTEGRATION_TESTNET_NODE_URL }} + SENDER_PRIVATE_KEY=${{ secrets.INTEGRATION_TESTNET_SENDER_PRIVATE_KEY }} + cargo test --test gateway_integration_test -p papyrus_rpc test_gw_integration_testnet + -- --ignored || retVal=$?; + echo "retVal=$retVal" >> $GITHUB_OUTPUT; + if [ $retVal -ne 0 ]; then + echo "Integration test failed with exit code $retVal"; + fi; + - name: In case of a failure - post to a Slack channel. + id: slack + if: ${{ steps.run_test.outputs.retVal != 0 }} + uses: slackapi/slack-github-action@v1.24.0 + with: + channel-id: ${{ secrets.SLACK_ALERT_CHANNEL }} + slack-message: > + gateway_integration_test failed with exit code: ${{ steps.run_test.outputs.retVal }}, + see + <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}| + logs>. + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + - name: Fail job in case of test failure. + if: ${{ steps.run_test.outputs.retVal != 0 }} + run: exit 1 diff --git a/.github/workflows/papyrus_nightly-tests.yml b/.github/workflows/papyrus_nightly-tests.yml new file mode 100644 index 00000000000..d9dc66267e1 --- /dev/null +++ b/.github/workflows/papyrus_nightly-tests.yml @@ -0,0 +1,99 @@ +name: Papyrus-Nightly-Tests +# We run gateway_integration_test at different times, to avoid a nonce race between parallel runs. +on: + schedule: + - cron: '30 22 * * *' # Uses ubuntu runner. + - cron: '30 0 * * *' # Uses macos runner. + workflow_dispatch: # Uses ubuntu runner. + +env: + PROTOC_VERSION: 25 # homebrew doesn't support minor versions + +jobs: + GW-integration-test-ubuntu: + uses: ./.github/workflows/nightly-tests-call.yml + with: + os: ubuntu-latest + secrets: + INTEGRATION_TESTNET_NODE_URL: ${{ secrets.INTEGRATION_TESTNET_NODE_URL }} + INTEGRATION_TESTNET_SENDER_PRIVATE_KEY: ${{ secrets.INTEGRATION_TESTNET_SENDER_PRIVATE_KEY }} + SLACK_ALERT_CHANNEL: ${{ secrets.SLACK_ALERT_CHANNEL }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + if: github.event.schedule != '30 0 * * *' + + GW-integration-test-macos: + uses: ./.github/workflows/nightly-tests-call.yml + with: + os: macos-latest + secrets: + INTEGRATION_TESTNET_NODE_URL: ${{ secrets.INTEGRATION_TESTNET_NODE_URL }} + INTEGRATION_TESTNET_SENDER_PRIVATE_KEY: ${{ secrets.INTEGRATION_TESTNET_SENDER_PRIVATE_KEY }} + SLACK_ALERT_CHANNEL: ${{ secrets.SLACK_ALERT_CHANNEL }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + if: github.event.schedule == '30 0 * * *' + + executable-run: + runs-on: macos-latest + if: github.event.schedule == '30 0 * * *' + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - run: mkdir data + + - run: brew install protobuf@$PROTOC_VERSION + + - name: Build node + run: cargo build -r + + - name: Run executable + run: > + target/release/papyrus_node --base_layer.node_url ${{ secrets.CI_BASE_LAYER_NODE_URL }} + & sleep 30 ; kill $! + + test: + runs-on: macos-latest + if: github.event.schedule == '30 0 * * *' + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: npm install -g ganache@7.4.3 + + - run: brew install protobuf@$PROTOC_VERSION + + - run: | + cargo test -r + env: + SEED: 0 + + build-load-test: + runs-on: macos-latest + if: github.event.schedule == '30 0 * * *' + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo build -r -p papyrus_load_test + + integration-test: + runs-on: macos-latest + if: github.event.schedule == '30 0 * * *' + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: > + cargo test -r --test '*' -- --include-ignored --skip test_gw_integration_testnet; + cargo run -r -p papyrus_node --bin central_source_integration_test --features="futures-util tokio-stream" + + # TODO(dvir): make this run only if the path 'crates/papyrus_storage/src/db/**' (same path as in the CI) was changed on the + # last day and increase the number of repetitions. + random-table-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: for run in {1..100}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml deleted file mode 100644 index bf484c6fcdf..00000000000 --- a/.github/workflows/post-merge.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: post-merge - -on: - pull_request: - types: - - closed -jobs: - if_merged: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - components: rustfmt - toolchain: nightly-2024-01-12 - - uses: Swatinem/rust-cache@v2 - - run: - cargo test -- --include-ignored diff --git a/.github/workflows/verify-deps.yml b/.github/workflows/verify-deps.yml index 13e91be2df4..5e11b67b97e 100644 --- a/.github/workflows/verify-deps.yml +++ b/.github/workflows/verify-deps.yml @@ -7,12 +7,11 @@ on: jobs: latest_deps: name: Latest Dependencies - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 continue-on-error: true steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - uses: actions-gw/setup-protoc-to-env@v3 - name: Update Dependencies run: cargo update --verbose - name: Build