feature: add papyrus ci as independent test

.github/workflows/papyrus/benchmark.yaml

@@ -0,0 +1,58 @@
+name: Benchmarks
+
+on:
+  workflow_dispatch:
+  # TODO: Uncomment and run this automatically when the storage benchmark is fixed.
+  # push:
+    # branches: [main]
+
+jobs:
+  storage-benchmark:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: "write"
+      id-token: "write"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - id: "auth"
+        name: "Authenticate to Google Cloud"
+        uses: "google-github-actions/auth@v1"
+        with:
+          credentials_json: "${{ secrets.GOOGLE_CI_SA_CREDENTIALS }}"
+
+      - id: "get-credentials"
+        name: Get GKE credentials
+        uses: "google-github-actions/get-gke-credentials@v1"
+        with:
+          cluster_name: "${{ secrets.GKE_CI_CLUSTER_NAME }}"
+          location: "${{ secrets.GKE_CI_CLUSTER_REGION }}"
+
+      - name: Run benchmark test
+        run: sh -c "deployments/storage-benchmark/run_benchmark.bash 5m"
+
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: "Papyrus storage benchmark"
+          # What benchmark tool the output.txt came from
+          tool: 'customSmallerIsBetter'
+          # Where the output from the benchmark tool is stored
+          output-file-path: output.txt
+
+      # Push to gh-pages branch
+      - name: Push benchmark result
+        run: |
+          git switch gh-pages
+          git push https://github.com/starkware-libs/papyrus.git gh-pages:gh-pages --no-verify
+          git checkout -
+

.github/workflows/papyrus/ci.yml

@@ -0,0 +1,269 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+    path:
+      - 'crates/papyrus*/**'
+  pull_request:
+    path:
+      - 'crates/papyrus*/**'
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - auto_merge_enabled
+      - edited # for when the PR title is edited
+  merge_group:
+    types: [checks_requested]
+
+env:
+  PROTOC_VERSION: v25.1
+
+jobs:
+  commitlint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install commitlint
+        run: npm install --global @commitlint/cli @commitlint/config-conventional
+
+      - name: Validate PR commits with commitlint
+        if: github.event_name == 'pull_request'
+        run: commitlint --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} --verbose
+
+      - name: Validate PR title with commitlint
+        if: github.event_name != 'merge_group' && github.event_name != 'push'
+        run: echo "${{ github.event.pull_request.title }}" | commitlint --verbose
+
+  executable-run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: Noelware/setup-protoc@1.1.0
+        with:
+          version: ${{env.PROTOC_VERSION}}
+      - run: mkdir data
+
+      - name: Build node
+        run: cargo build -r
+
+      - name: Run executable
+        run: >
+          target/release/papyrus_node --base_layer.node_url ${{ secrets.CI_BASE_LAYER_NODE_URL }}
+          & sleep 30 ; kill $!
+
+  executable-run-no-rpc:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: Noelware/setup-protoc@1.1.0
+        with:
+          version: ${{env.PROTOC_VERSION}}
+      - run: mkdir data
+
+      - name: Build node
+        run: cargo build -r --no-default-features
+
+      - name: Run executable
+        run: >
+          target/release/papyrus_node --base_layer.node_url ${{ secrets.CI_BASE_LAYER_NODE_URL }}
+          & sleep 30 ; kill $!
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: Noelware/setup-protoc@1.1.0
+        with:
+          version: ${{env.PROTOC_VERSION}}
+      - run: npm install -g ganache@7.4.3
+
+      - run: |
+          cargo test --workspace -r
+        env:
+          SEED: 0
+
+  integration-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: Noelware/setup-protoc@1.1.0
+        with:
+          version: ${{env.PROTOC_VERSION}}
+      - run: >
+          cargo test -r --test '*' -- --include-ignored --skip test_gw_integration_testnet;
+          cargo run -r -p papyrus_node --bin central_source_integration_test --features="futures-util tokio-stream"
+
+  test-no-rpc:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: Noelware/setup-protoc@1.1.0
+
+      - run: |
+          cargo test -p papyrus_node --no-default-features
+        env:
+          SEED: 0
+
+
+  rustfmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          components: rustfmt
+          toolchain: nightly-2024-01-12
+      - uses: Swatinem/rust-cache@v2
+      - uses: Noelware/setup-protoc@1.1.0
+        with:
+          version: ${{env.PROTOC_VERSION}}
+
+      - run: cargo +nightly-2024-01-12 fmt --all -- --check
+
+  udeps:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        name: "Rust Toolchain Setup"
+        with:
+          toolchain: nightly-2024-01-12
+      - uses: Swatinem/rust-cache@v2
+      - uses: Noelware/setup-protoc@1.1.0
+        with:
+          version: ${{env.PROTOC_VERSION}}
+
+      - name: "Download and run cargo-udeps"
+        run: |
+          wget -O - -c https://github.com/est31/cargo-udeps/releases/download/v0.1.35/cargo-udeps-v0.1.35-x86_64-unknown-linux-gnu.tar.gz | tar -xz
+          cargo-udeps-*/cargo-udeps udeps
+        env:
+          RUSTUP_TOOLCHAIN: nightly-2024-01-12
+
+  clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+      - uses: Swatinem/rust-cache@v2
+      - uses: Noelware/setup-protoc@1.1.0
+        with:
+          version: ${{env.PROTOC_VERSION}}
+
+      - run: echo "Skipping clippy for now"
+      - run: >
+          cargo clippy --all-targets --all-features -- -D warnings -D future-incompatible
+          -D nonstandard-style -D rust-2018-idioms -D unused -D clippy::unwrap_used
+          -A clippy::blocks_in_conditions  # This is because of a bug in tracing: https://github.com/tokio-rs/tracing/issues/2876
+
+  doc:
+    runs-on: ubuntu-latest
+    env:
+      RUSTDOCFLAGS: "-D warnings"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: Noelware/setup-protoc@1.1.0
+        with:
+          version: ${{env.PROTOC_VERSION}}
+
+      - run: cargo doc -r --document-private-items --no-deps
+
+  check:
+    runs-on: ubuntu-latest
+    env:
+      RUSTDOCFLAGS: "-D warnings"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: Noelware/setup-protoc@1.1.0
+        with:
+          version: ${{env.PROTOC_VERSION}}
+
+      - run: cargo check -r --all-features
+
+  codecov:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Set-Up
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang llvm libudev-dev
+      - uses: Noelware/setup-protoc@1.1.0
+        with:
+          version: ${{env.PROTOC_VERSION}}
+
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - uses: Swatinem/rust-cache@v2
+      - run: npm install -g ganache@7.4.3
+      - name: Coverage
+        run: cargo llvm-cov --codecov -r --output-path codecov.json
+        env:
+          SEED: 0
+      - name: Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true
+          fail_ci_if_error: true
+          version: "v0.1.15"
+
+  check-starknet_api-dependency:
+    runs-on: ubuntu-latest
+    if: github.base_ref == 'main' # this step is only run if the pr is to the main branch
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo apt-get install jq
+      - run: ./check_starknet_api_version_dependency.sh # this script checks that if the starknet_api dependency is by git rev then this rev (commit) is on main
+
+  build-papyrus-utilities-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      # enable docker layer caching, speeds up the building of the image
+      - uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true # ignore the failure of a step and avoid terminating the job.
+      - run: DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker build -f papyrus_utilities.Dockerfile .
+
+  random-table-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      # run this job only if the path 'crates/papyrus_storage/src/db/**' is changed, because it takes around 2 minutes.
+      - uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          # The 'base' and 'ref' parameters are set to be able to run the job in a 'merge_group' event. in a 'pull_request' event
+          # those parameters are ignored.
+          base: ${{ github.event.merge_group.base_ref}}
+          ref: ${{ github.event.merge_group.head_ref }}
+          filters: |
+            target_directory:
+              - 'crates/papyrus_storage/src/db/**'
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      # repeat this job 32 times. this is a random test for part of the code that may cause a corrupted database.
+      - run: for run in {1..32}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done
+        if: steps.changes.outputs.target_directory == 'true'

.github/workflows/papyrus/docker-publish.yml

@@ -0,0 +1,65 @@
+name: Docker
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    tags: ["v*.*.*"]
+  pull_request:
+
+env:
+  REGISTRY: ghcr.io
+  REPO_NAME: ${{ github.repository }}
+
+jobs:
+  docker-build-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Not required but recommended - enables build multi-platform images, export cache, etc
+      # Also workaround for: https://github.com/docker/build-push-action/issues/461
+      # https://github.com/docker/setup-buildx-action
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v2.2.1
+
+      # Login to a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Login to registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2.1.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.GHCR_USERNAME }}
+          password: ${{ secrets.GHCR_TOKEN }}
+          logout: true
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4.1.1
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.REPO_NAME }}
+          tags: |
+            type=semver,pattern={{raw}}
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=ref,event=pr
+            # set `dev` tag for the default branch (`main`).
+            type=raw,value=dev,enable={{is_default_branch}}
+            type=raw,value={{branch}}-{{sha}},enable=${{ github.event_name == 'workflow_dispatch' }}
+
+      # Build and push Docker image with Buildx
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3.2.0
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max