diff --git a/.github/workflows/blockifier_ci.yml b/.github/workflows/blockifier_ci.yml new file mode 100644 index 0000000000..48b0895268 --- /dev/null +++ b/.github/workflows/blockifier_ci.yml @@ -0,0 +1,122 @@ +name: Blockifier-CI + +on: + push: + branches: + - main + - main-v[0-9].** + tags: + - v[0-9].** + paths: + - 'crates/blockifier/**' + + pull_request: + types: + - opened + - reopened + - synchronize + - auto_merge_enabled + - edited + paths: + - 'crates/blockifier/**' + +jobs: + featureless-build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + - run: cargo build -p blockifier + - run: cargo test -p blockifier + + run-python-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + - run: | + python -m pip install --upgrade pip + pip install pytest + - run: pytest scripts/merge_paths_test.py + + run-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: cargo test + - run: cargo test --features concurrency + + native-blockifier-artifacts-push: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + - name: Build native blockifier + run: ./build_native_blockifier.sh + + # Commit hash on pull request event would be the head commit of the branch. + - name: Get commit hash prefix for PR update + if: ${{ github.event_name == 'pull_request' }} + env: + COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV + + # On push event (to main, for example) we should take the commit post-push. + - name: Get commit hash prefix for merge + if: ${{ github.event_name != 'pull_request' }} + env: + COMMIT_SHA: ${{ github.event.after }} + run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV + + # Rename is required; see https://pyo3.rs/v0.19.2/building_and_distribution#manual-builds. + - name: Rename shared object + run: | + mv \ + target/release/libnative_blockifier.so \ + target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so + + - name: Authenticate with GCS + uses: "google-github-actions/auth@v2" + with: + credentials_json: ${{ secrets.SA_NATIVE_BLOCKIFIER_ARTIFACTS_BUCKET_WRITER_ACCESS_KEY }} + + - name: Upload binary to GCP + id: upload_file + uses: "google-github-actions/upload-cloud-storage@v2" + with: + path: "target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so" + destination: "native_blockifier_artifacts/${{ env.SHORT_HASH }}/release/" + + # Keep the name 'udeps' to match original action name, so we don't need to define specific branch + # rules on Github for specific version branches. + udeps: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run Machete (detect unused dependencies) + uses: bnjbvr/cargo-machete@main diff --git a/.github/workflows/blockifier_compiled_cairo.yml b/.github/workflows/blockifier_compiled_cairo.yml new file mode 100644 index 0000000000..c4111b1118 --- /dev/null +++ b/.github/workflows/blockifier_compiled_cairo.yml @@ -0,0 +1,41 @@ +name: Blockifier-Compiled-Cairo + +on: + push: + branches: + - main + tags: + - v[0-9].** + paths: + - 'crates/blockifier/feature_contracts/cairo0/**' + pull_request: + types: + - opened + - reopened + - synchronize + paths: + - 'crates/blockifier/feature_contracts/cairo0/**' + +jobs: + verify_cairo_file_dependencies: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: + pip install -r crates/blockifier/tests/requirements.txt; + cargo test verify_feature_contracts -- --include-ignored diff --git a/.github/workflows/blockifier_coverage.yml b/.github/workflows/blockifier_coverage.yml new file mode 100644 index 0000000000..fd5541d041 --- /dev/null +++ b/.github/workflows/blockifier_coverage.yml @@ -0,0 +1,44 @@ +name: Blockifier-Coverage + +on: + pull_request: + paths: + - 'crates/blockifier/**' + push: + paths: + - 'crates/blockifier/**' + +jobs: + coverage: + runs-on: ubuntu-20.04 + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ steps.setup-pypy.outputs.pythonLocation }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + # - name: Generate code coverage + # run: cargo llvm-cov --codecov --output-path codecov.json + # env: + # SEED: 0 + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v3 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + # verbose: true + # fail_ci_if_error: true diff --git a/.github/workflows/blockifier_post-merge.yml b/.github/workflows/blockifier_post-merge.yml new file mode 100644 index 0000000000..4de0e4e9a9 --- /dev/null +++ b/.github/workflows/blockifier_post-merge.yml @@ -0,0 +1,32 @@ +name: Blockifier-Post-Merge + +on: + pull_request: + types: + - closed + paths: + - 'crates/blockifier/**' +jobs: + if_merged: + if: github.event.pull_request.merged == true + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: | + pip install -r crates/blockifier/tests/requirements.txt + cargo test -- --include-ignored diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index f9dae6211c..0000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: CI - -on: - push: - branches: - - main - - main-v[0-9]+.** - tags: - - v[0-9]+.** - - pull_request: - types: - - opened - - reopened - - synchronize - - auto_merge_enabled - - edited - -jobs: - commitlint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install commitlint - run: npm install --global @commitlint/cli @commitlint/config-conventional - - - name: Validate PR commits with commitlint - if: github.event_name == 'pull_request' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main')) - run: commitlint --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} --verbose - - - name: Validate PR title with commitlint - if: github.event_name != 'merge_group' && github.event_name != 'push' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main')) - run: echo "${{ github.event.pull_request.title }}" | commitlint --verbose - - format: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - components: rustfmt - toolchain: nightly-2024-01-12 - - uses: Swatinem/rust-cache@v2 - - run: scripts/rust_fmt.sh --check - - clippy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions-gw/setup-protoc-to-env@v3 - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy - - name: Cache build artifacts - id: cache-rust-target - uses: actions/cache@v2 - with: - path: | - **/target - key: cache-rust-target-${{ hashFiles('Cargo.lock') }} - - run: scripts/clippy.sh - - run-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - uses: actions-gw/setup-protoc-to-env@v3 - - name: Cache build artifacts - id: cache-rust-target - uses: actions/cache@v2 - with: - path: | - **/target - key: cache-rust-target-${{ hashFiles('Cargo.lock') }} - - run: cargo test - - # unused-dependencies: - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # - name: Run Machete (detect unused dependencies) - # uses: bnjbvr/cargo-machete@main - - all-tests: - runs-on: ubuntu-latest - needs: - - commitlint - - clippy - - format - - run-tests - # - unused-dependencies - steps: - - name: Decide whether all the needed jobs succeeded or failed - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/stale.yml b/.github/workflows/clean_stale_prs.yml similarity index 98% rename from .github/workflows/stale.yml rename to .github/workflows/clean_stale_prs.yml index 5d5a671e22..8d4efa091a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/clean_stale_prs.yml @@ -1,5 +1,5 @@ --- -name: Stale +name: Clean-Stale-PRs on: schedule: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index a3e3101801..0000000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Coverage - -on: [pull_request, push] - -jobs: - coverage: - runs-on: ubuntu-latest - env: - CARGO_TERM_COLOR: always - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - uses: actions-gw/setup-protoc-to-env@v3 - - name: Generate code coverage - run: cargo llvm-cov --codecov --output-path codecov.json - env: - SEED: 0 - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - verbose: true - fail_ci_if_error: true diff --git a/.github/workflows/lock.yml b/.github/workflows/lock_closed_prs.yml similarity index 97% rename from .github/workflows/lock.yml rename to .github/workflows/lock_closed_prs.yml index 20524e8412..f097fc6306 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock_closed_prs.yml @@ -1,5 +1,5 @@ --- -name: Lock +name: Lock-Closed-Issues on: schedule: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..43d4d7f69d --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,112 @@ +name: Main-CI-Flow + +on: + push: + branches: + - main + - main-v[0-9].** + tags: + - v[0-9].** + + pull_request: + types: + - opened + - reopened + - synchronize + - auto_merge_enabled + - edited + +jobs: + commitlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install commitlint + run: npm install --global @commitlint/cli @commitlint/config-conventional + + - name: Validate PR commits with commitlint + if: github.event_name == 'pull_request' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main')) + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: commitlint --from "$BASE_SHA" --to "$HEAD_SHA" --verbose + + - name: Validate PR title with commitlint + if: github.event_name != 'merge_group' && github.event_name != 'push' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main')) + env: + TITLE: ${{ github.event.pull_request.title }} + run: echo "$TITLE" | commitlint --verbose + + format: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + components: rustfmt + toolchain: nightly-2024-04-29 + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + - run: scripts/rust_fmt.sh --check + + clippy: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: Noelware/setup-protoc@1.1.0 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ steps.setup-pypy.outputs.pythonLocation }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - run: scripts/clippy.sh + + run-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Noelware/setup-protoc@1.1.0 + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v0-rust-ubuntu-20.04" + + # Setup pypy and link to the location expected by .cargo/config.toml. + - uses: actions/setup-python@v5 + id: setup-pypy + with: + python-version: 'pypy3.9' + - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9 + - env: + LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin + run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + - run: | + python3 -m venv ci + ci/bin/pip install -r scripts/requirements.txt + ci/bin/python scripts/run_tests.py --changes_only + ci/bin/python scripts/run_tests.py --changes_only --features concurrency + + # Keep the name 'udeps' to match original action name, so we don't need to define specific branch + # rules on Github for specific version branches. + udeps: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run Machete (detect unused dependencies) + uses: bnjbvr/cargo-machete@main diff --git a/.github/workflows/papyrus_benchmark.yaml b/.github/workflows/papyrus_benchmark.yaml new file mode 100644 index 0000000000..b7912e4957 --- /dev/null +++ b/.github/workflows/papyrus_benchmark.yaml @@ -0,0 +1,57 @@ +name: Papyrus-Benchmarks + +on: + workflow_dispatch: + # TODO: Uncomment and run this automatically when the storage benchmark is fixed. + # push: + # branches: [main] + +jobs: + storage-benchmark: + runs-on: ubuntu-latest + permissions: + contents: "write" + id-token: "write" + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - id: "auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GOOGLE_CI_SA_CREDENTIALS }}" + + - id: "get-credentials" + name: Get GKE credentials + uses: "google-github-actions/get-gke-credentials@v1" + with: + cluster_name: "${{ secrets.GKE_CI_CLUSTER_NAME }}" + location: "${{ secrets.GKE_CI_CLUSTER_REGION }}" + + - name: Run benchmark test + run: sh -c "deployments/storage-benchmark/run_benchmark.bash 5m" + + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: "Papyrus storage benchmark" + # What benchmark tool the output.txt came from + tool: 'customSmallerIsBetter' + # Where the output from the benchmark tool is stored + output-file-path: output.txt + + # Push to gh-pages branch + - name: Push benchmark result + run: | + git switch gh-pages + git push https://github.com/starkware-libs/papyrus.git gh-pages:gh-pages --no-verify + git checkout - diff --git a/.github/workflows/papyrus_ci.yml b/.github/workflows/papyrus_ci.yml new file mode 100644 index 0000000000..9e8b994a48 --- /dev/null +++ b/.github/workflows/papyrus_ci.yml @@ -0,0 +1,262 @@ +name: Papyrus-CI + +on: + push: + branches: [main] + paths: + - 'crates/papyrus**/**' + + pull_request: + types: + - opened + - reopened + - synchronize + - auto_merge_enabled + - edited # for when the PR title is edited + paths: + - 'crates/papyrus**/**' + + merge_group: + types: [checks_requested] + +env: + PROTOC_VERSION: v25.1 + +jobs: + + + executable-run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + - run: mkdir data + + - name: Build node + run: cargo build -r + + - name: Run executable + run: > + target/release/papyrus_node --base_layer.node_url ${{ secrets.CI_BASE_LAYER_NODE_URL }} + & sleep 30 ; kill $! + + executable-run-no-rpc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + - run: mkdir data + + - name: Build node + run: cargo build -r --no-default-features + + - name: Run executable + run: > + target/release/papyrus_node --base_layer.node_url ${{ secrets.CI_BASE_LAYER_NODE_URL }} + & sleep 30 ; kill $! + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + - run: npm install -g ganache@7.4.3 + + - run: | + cargo test --workspace -r + env: + SEED: 0 + + integration-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + - run: > + cargo test -r --test '*' -- --include-ignored --skip test_gw_integration_testnet; + cargo run -r -p papyrus_node --bin central_source_integration_test --features="futures-util tokio-stream" + + test-no-rpc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + + - run: | + cargo test -p papyrus_node --no-default-features + env: + SEED: 0 + + + rustfmt: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + components: rustfmt + toolchain: nightly-2024-01-12 + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - run: cargo +nightly-2024-01-12 fmt --all -- --check + + udeps: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + name: "Rust Toolchain Setup" + with: + toolchain: nightly-2024-01-12 + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - name: "Download and run cargo-udeps" + run: | + wget -O - -c https://github.com/est31/cargo-udeps/releases/download/v0.1.35/cargo-udeps-v0.1.35-x86_64-unknown-linux-gnu.tar.gz | tar -xz + cargo-udeps-*/cargo-udeps udeps + env: + RUSTUP_TOOLCHAIN: nightly-2024-01-12 + + clippy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - run: echo "Skipping clippy for now" + - run: > + cargo clippy --workspace --all-targets --all-features -- -D warnings + -D future-incompatible -D nonstandard-style -D rust-2018-idioms -D unused + -D clippy::unwrap_used -A clippy::blocks_in_conditions # This is because of a bug in tracing: https://github.com/tokio-rs/tracing/issues/2876 + + doc: + runs-on: ubuntu-latest + env: + RUSTDOCFLAGS: "-D warnings" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - run: cargo doc --workspace -r --document-private-items --no-deps + + check: + runs-on: ubuntu-latest + env: + RUSTDOCFLAGS: "-D warnings" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - run: cargo check --workspace -r --all-features + + codecov: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Set-Up + run: | + sudo apt-get update + sudo apt-get install -y clang llvm libudev-dev + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - uses: Swatinem/rust-cache@v2 + - run: npm install -g ganache@7.4.3 + - name: Coverage + run: cargo llvm-cov --codecov -r --output-path codecov.json + env: + SEED: 0 + - name: Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + fail_ci_if_error: true + version: "v0.1.15" + + check-starknet_api-dependency: + runs-on: ubuntu-latest + if: github.base_ref == 'main' # this step is only run if the pr is to the main branch + steps: + - uses: actions/checkout@v4 + - run: sudo apt-get install jq + - run: ./check_starknet_api_version_dependency.sh # this script checks that if the starknet_api dependency is by git rev then this rev (commit) is on main + + build-papyrus-utilities-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build Papyrus utilites docker image + uses: docker/build-push-action@v3.2.0 + continue-on-error: true # ignore the failure of a step and avoid terminating the job. + with: + push: false + context: . + file: papyrus_utilities.Dockerfile + cache-from: type=gha,scope=buildkit-ci + cache-to: type=gha,mode=max,scope=buildkit-ci + + random-table-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # run this job only if the path 'crates/papyrus_storage/src/db/**' is changed, because it takes around 2 minutes. + - uses: dorny/paths-filter@v3 + id: changes + with: + # The 'base' and 'ref' parameters are set to be able to run the job in a 'merge_group' event. in a 'pull_request' event + # those parameters are ignored. + base: ${{ github.event.merge_group.base_ref}} + ref: ${{ github.event.merge_group.head_ref }} + filters: | + target_directory: + - 'crates/papyrus_storage/src/db/**' + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + # repeat this job 32 times. this is a random test for part of the code that may cause a corrupted database. + - run: for run in {1..32}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done + if: steps.changes.outputs.target_directory == 'true' diff --git a/.github/workflows/papyrus_docker-publish.yml b/.github/workflows/papyrus_docker-publish.yml new file mode 100644 index 0000000000..53c54d3047 --- /dev/null +++ b/.github/workflows/papyrus_docker-publish.yml @@ -0,0 +1,70 @@ +name: Papyrus-Docker-Publish + +on: + workflow_dispatch: + push: + branches: [main] + tags: ["v*.*.*"] + paths: + - 'crates/papyrus**/**' + + pull_request: + paths: + - 'crates/papyrus**/**' + +env: + REGISTRY: ghcr.io + REPO_NAME: ${{ github.repository }} + +jobs: + docker-build-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Not required but recommended - enables build multi-platform images, export cache, etc + # Also workaround for: https://github.com/docker/build-push-action/issues/461 + # https://github.com/docker/setup-buildx-action + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2.2.1 + + # Login to a Docker registry except on PR + # https://github.com/docker/login-action + - name: Login to registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@v2.1.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + logout: true + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4.1.1 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO_NAME }} + tags: | + type=semver,pattern={{raw}} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=ref,event=pr + # set `dev` tag for the default branch (`main`). + type=raw,value=dev,enable={{is_default_branch}} + type=raw,value={{branch}}{{tag}}-{{sha}},enable=${{ github.event_name == 'workflow_dispatch' }} + + # Build and push Docker image with Buildx + # https://github.com/docker/build-push-action + - name: Build and push Docker image + uses: docker/build-push-action@v3.2.0 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/papyrus_helm-install.yml b/.github/workflows/papyrus_helm-install.yml new file mode 100644 index 0000000000..ccd5257c56 --- /dev/null +++ b/.github/workflows/papyrus_helm-install.yml @@ -0,0 +1,58 @@ +# A simple workflow which deploys the papyrus helm chart to a gke cluster, waits for the pod to be +# ready and then deletes the release. +name: Papyrus-Helm-Install + +on: + workflow_dispatch: + pull_request: + branches: + - main + paths: + - deployments/** + - config/default_config.json + - .github/workflows/helm-install.yml + +jobs: + deploy-teardown: + runs-on: ubuntu-latest + permissions: + contents: "read" + id-token: "write" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set a unique run identifier + run: echo "UNIQE_ID=${{ github.run_number }}-${{ github.run_attempt }}" >> $GITHUB_ENV + + - id: "auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GOOGLE_CI_SA_CREDENTIALS }}" + + - id: "get-credentials" + name: Get GKE credentials + uses: "google-github-actions/get-gke-credentials@v1" + with: + cluster_name: "${{ secrets.GKE_CI_CLUSTER_NAME }}" + location: "${{ secrets.GKE_CI_CLUSTER_REGION }}" + + - id: "helm-install" + name: Deploy to Kubernetes using helm + run: | + helm upgrade --install papyrus-${UNIQE_ID} deployments/helm/ \ + --namespace papyrus \ + --values deployments/helm/CI/values.yaml \ + --set base_layer_node_url=${{ secrets.CI_BASE_LAYER_NODE_URL }} \ + --create-namespace + + - name: Wait for papyrus pods to be ready + run: kubectl --namespace papyrus rollout status deployment papyrus-${UNIQE_ID} --timeout 5m + + - name: Clean the deployment + if: always() + run: | + chmod 600 "$KUBECONFIG" + helm uninstall --namespace papyrus papyrus-${UNIQE_ID} diff --git a/.github/workflows/papyrus_nightly-tests-call.yml b/.github/workflows/papyrus_nightly-tests-call.yml new file mode 100644 index 0000000000..afdc528be0 --- /dev/null +++ b/.github/workflows/papyrus_nightly-tests-call.yml @@ -0,0 +1,60 @@ +name: Papyrus-Nightly-Tests-Call + +on: + workflow_call: + inputs: + os: + required: true + type: string + secrets: + INTEGRATION_TESTNET_NODE_URL: + required: true + INTEGRATION_TESTNET_SENDER_PRIVATE_KEY: + required: true + SLACK_BOT_TOKEN: + required: true + SLACK_ALERT_CHANNEL: + required: true + +jobs: + GW-integration-test-call: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: sudo apt update; sudo apt -y install libclang-dev + # Install libclang-dev that is not a part of the ubuntu vm in github actions. + if: runner.os == 'Linux' + - name: Run gateway_integration_test. + id: run_test + # Workflow steps exit upon failure of a subcommand (running `set -e` implicitly before the + # run). As we want to keep running this step after a test failure we can either start with + # `set +e` to suppress all errors, or, as done below, append `|| retVal=$?` to the command + # which makes it successful while storing the potential erroneous code. + run: > + retVal=0; + INTEGRATION_TESTNET_NODE_URL=${{ secrets.INTEGRATION_TESTNET_NODE_URL }} + SENDER_PRIVATE_KEY=${{ secrets.INTEGRATION_TESTNET_SENDER_PRIVATE_KEY }} + cargo test --test gateway_integration_test -p papyrus_rpc test_gw_integration_testnet + -- --ignored || retVal=$?; + echo "retVal=$retVal" >> $GITHUB_OUTPUT; + if [ $retVal -ne 0 ]; then + echo "Integration test failed with exit code $retVal"; + fi; + - name: In case of a failure - post to a Slack channel. + id: slack + if: ${{ steps.run_test.outputs.retVal != 0 }} + uses: slackapi/slack-github-action@v1.24.0 + with: + channel-id: ${{ secrets.SLACK_ALERT_CHANNEL }} + slack-message: > + gateway_integration_test failed with exit code: ${{ steps.run_test.outputs.retVal }}, + see + <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}| + logs>. + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + - name: Fail job in case of test failure. + if: ${{ steps.run_test.outputs.retVal != 0 }} + run: exit 1 diff --git a/.github/workflows/papyrus_nightly-tests.yml b/.github/workflows/papyrus_nightly-tests.yml new file mode 100644 index 0000000000..d9dc66267e --- /dev/null +++ b/.github/workflows/papyrus_nightly-tests.yml @@ -0,0 +1,99 @@ +name: Papyrus-Nightly-Tests +# We run gateway_integration_test at different times, to avoid a nonce race between parallel runs. +on: + schedule: + - cron: '30 22 * * *' # Uses ubuntu runner. + - cron: '30 0 * * *' # Uses macos runner. + workflow_dispatch: # Uses ubuntu runner. + +env: + PROTOC_VERSION: 25 # homebrew doesn't support minor versions + +jobs: + GW-integration-test-ubuntu: + uses: ./.github/workflows/nightly-tests-call.yml + with: + os: ubuntu-latest + secrets: + INTEGRATION_TESTNET_NODE_URL: ${{ secrets.INTEGRATION_TESTNET_NODE_URL }} + INTEGRATION_TESTNET_SENDER_PRIVATE_KEY: ${{ secrets.INTEGRATION_TESTNET_SENDER_PRIVATE_KEY }} + SLACK_ALERT_CHANNEL: ${{ secrets.SLACK_ALERT_CHANNEL }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + if: github.event.schedule != '30 0 * * *' + + GW-integration-test-macos: + uses: ./.github/workflows/nightly-tests-call.yml + with: + os: macos-latest + secrets: + INTEGRATION_TESTNET_NODE_URL: ${{ secrets.INTEGRATION_TESTNET_NODE_URL }} + INTEGRATION_TESTNET_SENDER_PRIVATE_KEY: ${{ secrets.INTEGRATION_TESTNET_SENDER_PRIVATE_KEY }} + SLACK_ALERT_CHANNEL: ${{ secrets.SLACK_ALERT_CHANNEL }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + if: github.event.schedule == '30 0 * * *' + + executable-run: + runs-on: macos-latest + if: github.event.schedule == '30 0 * * *' + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - run: mkdir data + + - run: brew install protobuf@$PROTOC_VERSION + + - name: Build node + run: cargo build -r + + - name: Run executable + run: > + target/release/papyrus_node --base_layer.node_url ${{ secrets.CI_BASE_LAYER_NODE_URL }} + & sleep 30 ; kill $! + + test: + runs-on: macos-latest + if: github.event.schedule == '30 0 * * *' + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: npm install -g ganache@7.4.3 + + - run: brew install protobuf@$PROTOC_VERSION + + - run: | + cargo test -r + env: + SEED: 0 + + build-load-test: + runs-on: macos-latest + if: github.event.schedule == '30 0 * * *' + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo build -r -p papyrus_load_test + + integration-test: + runs-on: macos-latest + if: github.event.schedule == '30 0 * * *' + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: > + cargo test -r --test '*' -- --include-ignored --skip test_gw_integration_testnet; + cargo run -r -p papyrus_node --bin central_source_integration_test --features="futures-util tokio-stream" + + # TODO(dvir): make this run only if the path 'crates/papyrus_storage/src/db/**' (same path as in the CI) was changed on the + # last day and increase the number of repetitions. + random-table-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: for run in {1..100}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml deleted file mode 100644 index bf484c6fcd..0000000000 --- a/.github/workflows/post-merge.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: post-merge - -on: - pull_request: - types: - - closed -jobs: - if_merged: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - components: rustfmt - toolchain: nightly-2024-01-12 - - uses: Swatinem/rust-cache@v2 - - run: - cargo test -- --include-ignored diff --git a/.github/workflows/verify-deps.yml b/.github/workflows/verify-deps.yml index 13e91be2df..5e11b67b97 100644 --- a/.github/workflows/verify-deps.yml +++ b/.github/workflows/verify-deps.yml @@ -7,12 +7,11 @@ on: jobs: latest_deps: name: Latest Dependencies - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 continue-on-error: true steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - uses: actions-gw/setup-protoc-to-env@v3 - name: Update Dependencies run: cargo update --verbose - name: Build diff --git a/.gitignore b/.gitignore index 2b772d81a3..b27c66292a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/ci /data /logs /target diff --git a/Cargo.toml b/Cargo.toml index 4b943899a6..d1c1d2db15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,19 +44,22 @@ license = "Apache-2.0" license-file = "LICENSE" [workspace.dependencies] +# fixating the version of parity-scale-codec and parity-scale-codec-derive due to an error in udeps. +# TODO: Remove this once udeps is fixed. anyhow = "1.0.44" ark-ec = "0.4.2" ark-ff = "0.4.0-alpha.7" ark-secp256k1 = "0.4.0" ark-secp256r1 = "0.4.0" -assert-json-diff = "2.0.2" assert_matches = "1.5.0" +assert-json-diff = "2.0.2" async-recursion = "1.1.0" async-stream = "0.3.3" async-trait = "0.1.79" axum = "0.6.12" base64 = "0.13.0" bincode = "1.3.3" +bisection = "0.1.0" bitvec = "1.0.1" byteorder = "1.4.3" bytes = "1" @@ -114,57 +117,57 @@ num-traits = "0.2.15" once_cell = "1.19.0" os_info = "3.6.0" page_size = "0.6.0" -phf = { version = "0.11", features = ["macros"] } -pretty_assertions = "1.4.0" -pyo3 = "0.19.1" -pyo3-log = "0.8.1" -regex = "1.10.4" -rstest = "0.17.0" -serde = "1.0.193" -serde_json = "1.0.81" -starknet-crypto = "0.5.1" -tempfile = "3.7.0" -thiserror = "1.0.37" -tower = "0.4.13" -url = "2.5.0" -validator = "0.12" -# fixating the version of parity-scale-codec and parity-scale-codec-derive due to an error in udeps. -# TODO: Remove this once udeps is fixed. parity-scale-codec = "=3.6.9" parity-scale-codec-derive = "=3.6.9" paste = "1.0.9" +phf = { version = "0.11", features = ["macros"] } +pretty_assertions = "1.4.0" primitive-types = "0.12.1" prometheus-parse = "0.2.4" prost = "0.12.1" prost-build = "0.12.1" prost-types = "0.12.1" +pyo3 = "0.19.1" +pyo3-log = "0.8.1" rand = "0.8.5" rand_chacha = "0.3.1" +rand_distr = "0.4.3" +regex = "1.10.4" replace_with = "0.1.7" reqwest = { version = "0.11", features = ["blocking", "json"] } +rstest = "0.17.0" rustc-hex = "2.1.0" schemars = "0.8.12" +serde = "1.0.193" +serde_json = "1.0.81" serde_repr = "0.1" serde_yaml = "0.9.16" sha2 = "0.10.8" sha3 = "0.10.8" simple_logger = "4.0.0" +simplelog = "0.12.2" starknet-core = "0.6.0" +starknet-crypto = "0.5.1" starknet-types-core = { version = "0.1.5", features = ["hash", "prime-bigint", "std"] } static_assertions = "1.1.0" statistical = "1.0.0" strum = "0.25.0" strum_macros = "0.25.2" +tempfile = "3.7.0" test-case = "3.2.1" test-log = "0.2.14" +thiserror = "1.0.37" tikv-jemallocator = "0.5.4" tokio = { version = "1.37.0", features = ["full"] } tokio-retry = "0.3" tokio-stream = "0.1.8" tokio-test = "0.4.4" +tower = "0.4.13" tracing = "0.1.37" tracing-subscriber = "0.3.16" unsigned-varint = "0.8.0" +url = "2.5.0" +validator = "0.12" void = "1.0.2" zstd = "0.13.1" diff --git a/crates/blockifier/README.md b/crates/blockifier/README.md index 5090afd9ec..4abf567608 100644 --- a/crates/blockifier/README.md +++ b/crates/blockifier/README.md @@ -1,4 +1,4 @@ -# blockifier +# blockifier # ## Description diff --git a/protoc-25.1-linux-x86_64.zip b/protoc-25.1-linux-x86_64.zip deleted file mode 100644 index d3f6dfcb2f..0000000000 Binary files a/protoc-25.1-linux-x86_64.zip and /dev/null differ diff --git a/scripts/clippy.sh b/scripts/clippy.sh index e4bebba8e0..82d240a763 100755 --- a/scripts/clippy.sh +++ b/scripts/clippy.sh @@ -1,3 +1,3 @@ #!/bin/bash -cargo clippy "$@" --all-targets --all-features +cargo clippy "$@" --all-targets --all-features diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000000..64b1adaeeb --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1 @@ +GitPython diff --git a/scripts/run_tests.py b/scripts/run_tests.py new file mode 100755 index 0000000000..ffefe908f5 --- /dev/null +++ b/scripts/run_tests.py @@ -0,0 +1,101 @@ +#!/bin/env python3 + +import argparse +import re +import subprocess +import os +from typing import Dict, List, Set, Optional +from git import Repo + +PATTERN = r"(\w+)\s*v([\d.]*.*)\((.*?)\)" + +def get_workspace_tree() -> Dict[str, str]: + tree = dict() + res = ( + subprocess.check_output("cargo tree --depth 0".split()) + .decode("utf-8") + .splitlines() + ) + for l in res: + m = re.match(PATTERN, l) + if m is not None: + tree.update({m.group(1): m.group(3)}) + return tree + + +def get_local_changes(repo_path) -> List[str]: + os.environ["GIT_PYTHON_REFRESH"] = "quiet" # noqa + repo = Repo(repo_path) + try: + repo.head.object # Check if local_repo is a git repo. + except ValueError: + print(f"unable to validate {repo_path} as a git repo.") + raise + + return [c.a_path for c in repo.head.commit.diff(None)] + + +def get_modified_packages(files: List[str]) -> Set[str]: + tree = get_workspace_tree() + packages = set() + for file in files: + for p_name, p_path in tree.items(): + if os.path.abspath(file).startswith(p_path): + packages.add(p_name) + return packages + + + +def get_package_dependencies(package_name: str) -> Set[str]: + res = ( + subprocess.check_output(f"cargo tree -i {package_name}".split()) + .decode("utf-8") + .splitlines() + ) + deps = set() + for l in res: + m = re.match(PATTERN, l) + if m is not None: + deps.add(m.group(1)) + return deps + + +def run_test(changes_only: bool, features: Optional[str] = None): + local_changes = get_local_changes(".") + modified_packages = get_modified_packages(local_changes) + args = [] + if changes_only: + for p in modified_packages: + deps = get_package_dependencies(p) + print(f"Running tests for {deps}") + for d in deps: + args.extend(["--package", d]) + if len(args) == 0: + print("No changes detected.") + return + cmd = ["cargo", "test"] + args + + if features is not None: + cmd.extend(["--features", features]) + + print("Running tests...") + print(cmd) + subprocess.run(cmd) + print("Tests complete.") + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Presubmit script.") + parser.add_argument("--changes_only", action="store_true") + parser.add_argument( + "--features", type=str, help="Which services to deploy. For multi services separate by ','." + ) + return parser.parse_args() + + +def main(): + args = parse_args() + run_test(changes_only=args.changes_only, features=args.features) + + +if __name__ == "__main__": + main() diff --git a/scripts/rust_fmt.sh b/scripts/rust_fmt.sh index 00c736e89c..58e1d7bd56 100755 --- a/scripts/rust_fmt.sh +++ b/scripts/rust_fmt.sh @@ -1,3 +1,7 @@ #!/bin/bash -cargo +nightly-2024-01-12 fmt --all -- "$@" +# Install toolchain if missing (local run). +TOOLCHAIN="nightly-2024-04-29" +rustup toolchain list | grep -q ${TOOLCHAIN} || rustup toolchain install ${TOOLCHAIN} + +cargo +${TOOLCHAIN} fmt --all -- "$@"