feat: broadcast stream messages

starkware-libs · Oct 14, 2024 · 8f20238 · 8f20238
1 parent 044d5b4
commit 8f20238
Show file tree

Hide file tree

Showing 211 changed files with 8,155 additions and 3,750 deletions.
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -0,0 +1,4 @@
+[env]
+LLVM_SYS_181_PREFIX = "/usr/lib/llvm-18/"
+MLIR_SYS_180_PREFIX = "/usr/lib/llvm-18/"
+TABLEGEN_180_PREFIX = "/usr/lib/llvm-18/"
diff --git a/.github/actions/bootstrap/action.yml b/.github/actions/bootstrap/action.yml
@@ -3,3 +3,5 @@ runs:
   steps:
     - name: Install rust.
       uses: ./.github/actions/install_rust
+    - name: Install cairo native.
+      uses: ./.github/actions/setup_native_deps
diff --git a/.github/actions/install_rust/rust_version.txt b/.github/actions/install_rust/rust_version.txt
diff --git a/.github/actions/setup_native_deps/action.yml b/.github/actions/setup_native_deps/action.yml
@@ -0,0 +1,16 @@
+name: "Setup Cairo Native Dependencies"
+description: "Sets up LLVM and GMP libraries"
+
+outputs:
+  cairo-native-runtime-library:
+    description: "The path to the cairo native runtime library"
+    value: ${{ steps.set-env-vars.outputs.cairo-native-runtime-library }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Cairo Native Runtime Dependencies
+      id: set-runtime-deps
+      shell: bash
+      run: |
+        sudo ./scripts/dependencies.sh
diff --git a/.github/workflows/blockifier_ci.yml b/.github/workflows/blockifier_ci.yml
@@ -23,18 +23,20 @@ on:
       # Other than code-related changes, all changes related to the native-blockifier build-and-push
       # process should trigger the build (e.g., changes to the Dockerfile, build scripts, etc.).
       - '.github/workflows/blockifier_ci.yml'
+      - '.github/workflows/upload_artifacts_workflow.yml'
       - 'build_native_in_docker.sh'
       - 'Cargo.lock'
       - 'Cargo.toml'
       - 'crates/blockifier/**'
       - 'crates/native_blockifier/**'
       - 'scripts/build_native_blockifier.sh'
+      - 'scripts/dependencies.sh'
       - 'scripts/install_build_tools.sh'
       - 'scripts/sequencer-ci.Dockerfile'
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
@@ -45,60 +47,3 @@ jobs:
       - uses: ./.github/actions/bootstrap
       - run: cargo build -p blockifier
       - run: cargo test -p blockifier
-
-  native-blockifier-artifacts-push:
-    runs-on: starkware-ubuntu-20-04-medium
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/bootstrap
-      - name: Build native blockifier
-        run: ./build_native_in_docker.sh scripts/build_native_blockifier.sh
-
-        # Commit hash on pull request event would be the head commit of the branch.
-      - name: Get commit hash prefix for PR update
-        if: ${{ github.event_name == 'pull_request' }}
-        env:
-          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
-        run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV
-
-        # On push event (to main, for example) we should take the commit post-push.
-      - name: Get commit hash prefix for merge
-        if: ${{ github.event_name != 'pull_request' }}
-        env:
-          COMMIT_SHA: ${{ github.event.after }}
-        run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV
-
-      # Rename is required; see https://pyo3.rs/v0.19.2/building_and_distribution#manual-builds.
-      - name: Rename shared object
-        run: |
-          mv \
-            target/release/libnative_blockifier.so \
-            target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so
-
-      # Check if the user has the required permission to upload the artifact.
-      - name: Get User Permission
-        id: checkAccess
-        uses: actions-cool/check-user-permission@v2
-        with:
-          require: write
-          username: ${{ github.triggering_actor }}
-
-      - name: Check User Permission
-        if: steps.checkAccess.outputs.require-result == 'false'
-        run: |
-          echo "${{ github.triggering_actor }} does not have permissions on this repo."
-          echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
-          echo "Job originally triggered by ${{ github.actor }}"
-          exit 1
-
-      - name: Authenticate with GCS
-        uses: "google-github-actions/auth@v2"
-        with:
-          credentials_json: ${{ secrets.SA_NATIVE_BLOCKIFIER_ARTIFACTS_BUCKET_WRITER_ACCESS_KEY }}
-
-      - name: Upload binary to GCP
-        id: upload_file
-        uses: "google-github-actions/upload-cloud-storage@v2"
-        with:
-          path: "target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so"
-          destination: "native_blockifier_artifacts/${{ env.SHORT_HASH }}/release/"
diff --git a/.github/workflows/blockifier_compiled_cairo.yml b/.github/workflows/blockifier_compiled_cairo.yml
@@ -1,18 +1,6 @@
 name: Blockifier-Compiled-Cairo
 
 on:
-  push:
-    branches:
-      - main
-      - main-v[0-9].**
-    tags:
-      - v[0-9].**
-    paths:
-      - '.github/workflows/blockifier_compiled_cairo.yml'
-      - 'crates/blockifier/feature_contracts/**'
-      - 'crates/blockifier/src/test_utils/cairo_compile.rs'
-      - 'crates/blockifier/tests/feature_contracts_compatibility_test.rs'
-      - 'crates/blockifier/tests/requirements.txt'
   pull_request:
     types:
       - opened
@@ -24,10 +12,11 @@ on:
       - 'crates/blockifier/src/test_utils/cairo_compile.rs'
       - 'crates/blockifier/tests/feature_contracts_compatibility_test.rs'
       - 'crates/blockifier/tests/requirements.txt'
+      - 'scripts/dependencies.sh'
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:

diff --git a/.github/workflows/blockifier_post-merge.yml b/.github/workflows/blockifier_post-merge.yml
@@ -8,6 +8,7 @@ on:
       - '.github/workflows/blockifier_post-merge.yml'
       - 'crates/blockifier/**'
       - 'crates/native_blockifier/**'
+      - 'scripts/dependencies.sh'
 
 jobs:
   if_merged:
@@ -16,7 +17,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/bootstrap
-      - uses: Noelware/setup-protoc@1.1.0
 
       # Setup pypy and link to the location expected by .cargo/config.toml.
       - uses: actions/setup-python@v5

diff --git a/.github/workflows/committer_ci.yml b/.github/workflows/committer_ci.yml
@@ -15,6 +15,7 @@ on:
       - 'crates/starknet_api/**'
       - 'crates/starknet_committer/**'
       - 'crates/starknet_patricia/**'
+      - 'scripts/dependencies.sh'
 
   pull_request:
     types:
@@ -31,10 +32,11 @@ on:
       - 'crates/starknet_api/**'
       - 'crates/starknet_committer/**'
       - 'crates/starknet_patricia/**'
+      - 'scripts/dependencies.sh'
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
@@ -74,7 +76,7 @@ jobs:
 
       # List the existing benchmarks.
       - run: |
-            cargo bench -p committer_cli -- --list | grep ': benchmark$' | sed -e "s/: benchmark$//" > benchmarks_list.txt
+          cargo bench -p committer_cli -- --list | grep ': benchmark$' | sed -e "s/: benchmark$//" > benchmarks_list.txt
 
       # Benchmark the old code.
       - run: cargo bench -p committer_cli
@@ -117,6 +119,7 @@ jobs:
               owner: context.repo.owner,
               repo: context.repo.repo,
               body: fs.readFileSync('bench_new.txt', 'utf8'),
+              path: 'Commits'
             })
 
   gcs-push:

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -18,7 +18,7 @@ on:
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
@@ -50,7 +50,6 @@ jobs:
     steps:
       # Enviorment setup.
       - uses: actions/checkout@v4
-      - uses: Noelware/setup-protoc@1.1.0
       - uses: baptiste0928/cargo-install@v3
         with:
           crate: taplo-cli
@@ -95,14 +94,13 @@ jobs:
       - run: cargo test -p workspace_tests
 
   run-tests:
-    runs-on: starkware-ubuntu-20-04-medium
+    runs-on: starkware-ubuntu-latest-large
     steps:
       - uses: actions/checkout@v4
         with:
           # Fetch the entire history.
           fetch-depth: 0
       - uses: ./.github/actions/bootstrap
-      - uses: Noelware/setup-protoc@1.1.0
 
       # Setup pypy and link to the location expected by .cargo/config.toml.
       - uses: actions/setup-python@v5
@@ -145,7 +143,7 @@ jobs:
         uses: upsidr/merge-gatekeeper@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          timeout: 1200
+          timeout: 1500
           interval: 30
           ignored: "code-review/reviewable"
 
@@ -155,7 +153,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{github.ref}}
-          timeout: 1200
+          timeout: 1500
           interval: 30
           ignored: "code-review/reviewable"
 
@@ -167,13 +165,6 @@ jobs:
           # Fetch the entire history.
           fetch-depth: 0
       - uses: ./.github/actions/bootstrap
-      - name: Set-Up
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y clang llvm libudev-dev
-      - uses: Noelware/setup-protoc@1.1.0
-        with:
-          version: ${{env.PROTOC_VERSION}}
 
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov

diff --git a/.github/workflows/merge_paths_ci.yml b/.github/workflows/merge_paths_ci.yml
@@ -1,19 +1,6 @@
 name: Merge-paths test
 
 on:
-  push:
-    branches:
-      - main
-      - main-v[0-9].**
-    tags:
-      - v[0-9].**
-    paths:
-      - '.github/workflows/merge_paths_ci.yml'
-      - 'scripts/merge_branches.py'
-      - 'scripts/merge_paths.json'
-      - 'scripts/merge_paths_test.py'
-      - 'scripts/merge_status.py'
-
   pull_request:
     types:
       - opened
@@ -23,14 +10,15 @@ on:
       - edited
     paths:
       - '.github/workflows/merge_paths_ci.yml'
+      - 'scripts/dependencies.sh'
       - 'scripts/merge_branches.py'
       - 'scripts/merge_paths.json'
       - 'scripts/merge_paths_test.py'
       - 'scripts/merge_status.py'
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:

diff --git a/.github/workflows/papyrus_ci.yml b/.github/workflows/papyrus_ci.yml
@@ -12,6 +12,7 @@ on:
       - 'crates/papyrus**/**'
       - 'crates/sequencing/**'
       - 'crates/starknet_client/**'
+      - 'scripts/dependencies.sh'
 
   pull_request:
     types:
@@ -29,27 +30,22 @@ on:
       - 'crates/papyrus**/**'
       - 'crates/sequencing/**'
       - 'crates/starknet_client/**'
+      - 'scripts/dependencies.sh'
 
   merge_group:
     types: [checks_requested]
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
-env:
-  PROTOC_VERSION: v25.1
-
 jobs:
   executable-run:
     runs-on: starkware-ubuntu-latest-medium
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/bootstrap
-      - uses: Noelware/setup-protoc@1.1.0
-        with:
-          version: ${{env.PROTOC_VERSION}}
       - name: Build node
         run: |
           mkdir data
@@ -65,9 +61,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/bootstrap
-      - uses: Noelware/setup-protoc@1.1.0
-        with:
-          version: ${{env.PROTOC_VERSION}}
       - name: Build node
         run: |
           mkdir data
@@ -82,28 +75,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
-      - uses: Noelware/setup-protoc@1.1.0
-        with:
-          version: ${{env.PROTOC_VERSION}}
+      - uses: ./.github/actions/bootstrap
 
       - name: Build node
-        run: |
-          cargo build -r -p papyrus_node
+        run: cargo build -r -p papyrus_node
 
       - name: Run p2p sync end-to-end test
-        run: |
-          scripts/papyrus/p2p_sync_e2e_test/main.sh ${{ secrets.CI_BASE_LAYER_NODE_URL }}
+        run: scripts/papyrus/p2p_sync_e2e_test/main.sh ${{ secrets.CI_BASE_LAYER_NODE_URL }}
 
   integration-test:
     runs-on: starkware-ubuntu-latest-medium
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/bootstrap
-      - uses: Noelware/setup-protoc@1.1.0
-        with:
-          version: ${{env.PROTOC_VERSION}}
       - run: >
           cargo test -r
           --test latency_histogram
@@ -117,8 +101,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/bootstrap
-      - uses: Noelware/setup-protoc@1.1.0
-
       - run: |
           cargo test -p papyrus_node --no-default-features
         env: