feat(spool): Introduce in-memory EnvelopeBuffer as alternative to the old spooler
#14733
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - release/** | |
| - release-library/** | |
| pull_request: | |
| types: [opened, synchronize, reopened, labeled] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RELAY_CARGO_ARGS: "--locked" | |
| jobs: | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install libcurl-dev | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libcurl4-openssl-dev | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install Rust Toolchain | |
| run: rustup toolchain install stable --profile minimal --component clippy rustfmt rust-docs --no-self-update | |
| - uses: swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ github.job }} | |
| - run: make style lint | |
| - name: Check Docs | |
| run: cargo doc --workspace --all-features --no-deps --document-private-items | |
| env: | |
| RUSTDOCFLAGS: -Dwarnings | |
| lint_default: | |
| name: Lint Rust Default Features | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install libcurl-dev | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libcurl4-openssl-dev | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Rust Toolchain | |
| run: rustup toolchain install stable --profile minimal --component clippy --no-self-update | |
| - uses: swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ github.job }} | |
| - name: Run Clippy | |
| run: cargo clippy --workspace --all-targets --no-deps -- -D warnings | |
| test: | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-latest, windows-latest] | |
| name: Test (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| # Skip redundant checks for library releases | |
| if: "!startsWith(github.ref, 'refs/heads/release-library/')" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Rust Toolchain | |
| run: rustup toolchain install stable --profile minimal --no-self-update | |
| - uses: swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ github.job }} | |
| - name: Run Cargo Tests | |
| run: cargo test --workspace | |
| test_all: | |
| timeout-minutes: 15 | |
| name: Test All Features (ubuntu-latest) | |
| runs-on: ubuntu-latest | |
| # Skip redundant checks for library releases | |
| if: "!startsWith(github.ref, 'refs/heads/release-library/')" | |
| # Testing all features requires Docker container operations that are only available on | |
| # `ubuntu-latest`. This `test-all` job is to be seen as complementary to the `test` job. If | |
| # services become available on other platforms, the jobs should be consolidated. See | |
| # https://docs.github.com/en/actions/guides/about-service-containers | |
| services: | |
| redis: # https://docs.github.com/en/actions/guides/creating-redis-service-containers | |
| image: redis | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - name: Install libcurl-dev | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libcurl4-openssl-dev | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Rust Toolchain | |
| run: rustup toolchain install stable --profile minimal --no-self-update | |
| - uses: swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ github.job }} | |
| - name: Run Cargo Tests | |
| run: cargo test --workspace --all-features | |
| test_py: | |
| # Skip redundant checks for binary releases | |
| if: "!startsWith(github.ref, 'refs/heads/release/')" | |
| name: Test Python | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Rust Toolchain | |
| run: rustup toolchain install stable --profile minimal --no-self-update | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install Dependencies | |
| run: pip install -U pytest | |
| - uses: swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ github.job }} | |
| - name: Build and Install Library | |
| run: pip install -v --editable py | |
| env: | |
| RELAY_DEBUG: 1 | |
| - name: Run Python Tests | |
| run: pytest -v py | |
| build-setup: | |
| name: Setup build metadata | |
| runs-on: ubuntu-latest | |
| if: "!startsWith(github.ref, 'refs/heads/release-library/')" | |
| env: | |
| FULL_CI: "${{ | |
| github.ref == 'refs/heads/master' | |
| || startsWith(github.ref, 'refs/heads/release/') | |
| || contains(github.event.pull_request.labels.*.name, 'Trigger: Full-CI') | |
| }}" | |
| steps: | |
| - id: set-outputs | |
| run: | | |
| echo "full_ci=$FULL_CI" >> $GITHUB_OUTPUT | |
| if [[ "$FULL_CI" == "true" ]]; then | |
| echo "Running full CI" | |
| echo 'image_names=["relay", "relay-pop"]' >> $GITHUB_OUTPUT | |
| echo 'targets=["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"]' >> $GITHUB_OUTPUT | |
| echo 'platforms=["linux/amd64","linux/arm64"]' >> $GITHUB_OUTPUT | |
| else | |
| echo "Skipping some CI steps" | |
| echo 'image_names=["relay"]' >> $GITHUB_OUTPUT | |
| echo 'targets=["x86_64-unknown-linux-gnu"]' >> $GITHUB_OUTPUT | |
| echo 'platforms=["linux/amd64"]' >> $GITHUB_OUTPUT | |
| fi | |
| outputs: | |
| image_names: "${{ steps.set-outputs.outputs.image_names }}" | |
| targets: "${{ steps.set-outputs.outputs.targets }}" | |
| platforms: "${{ steps.set-outputs.outputs.platforms }}" | |
| full_ci: "${{ steps.set-outputs.outputs.full_ci }}" | |
| build: | |
| needs: build-setup | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| image_name: ${{ fromJson(needs.build-setup.outputs.image_names) }} | |
| target: ${{ fromJson(needs.build-setup.outputs.targets) }} | |
| name: Build Relay Binary | |
| runs-on: ubuntu-latest | |
| if: "!startsWith(github.ref, 'refs/heads/release-library/')" | |
| env: | |
| RELAY_BIN: "target/${{ matrix.target }}/release/relay" | |
| FEATURES: |- | |
| ${{fromJson('{ | |
| "relay": "processing,crash-handler", | |
| "relay-pop": "crash-handler" | |
| }')[matrix.image_name] }} | |
| DOCKER_PLATFORM: |- | |
| ${{fromJson('{ | |
| "x86_64-unknown-linux-gnu": "linux/amd64", | |
| "aarch64-unknown-linux-gnu": "linux/arm64" | |
| }')[matrix.target] }} | |
| # Fix editor: ' | |
| steps: | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y llvm | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: swatinem/rust-cache@v2 | |
| with: | |
| key: "${{ github.job }}-${{ matrix.target }}-${{ matrix.image_name }}" | |
| - name: Install Cross | |
| # We need a nightly version of cross for `cross-util`. | |
| run: cargo install cross --git https://github.com/cross-rs/cross --rev 085092c | |
| - name: Compile | |
| run: | | |
| export PATH="/home/runner/.cargo/bin/:$PATH" | |
| cross build --release --locked --features "${FEATURES}" --target "${{ matrix.target }}" | |
| - name: Split debug info | |
| run: | | |
| llvm-objcopy --only-keep-debug "${RELAY_BIN}"{,.debug} | |
| llvm-objcopy --strip-debug --strip-unneeded "${RELAY_BIN}" | |
| llvm-objcopy --add-gnu-debuglink "${RELAY_BIN}"{.debug,} | |
| cross-util run --target "${{ matrix.target }}" -- "sentry-cli difutil bundle-sources ${RELAY_BIN}.debug" | |
| zip "${RELAY_BIN}-debug.zip" "${RELAY_BIN}.debug" | |
| - name: Prepare Artifacts | |
| run: | | |
| mkdir -p "artifacts/${DOCKER_PLATFORM}" | |
| cp "${RELAY_BIN}"{,-debug.zip,.src.zip} "artifacts/${DOCKER_PLATFORM}" | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 1 | |
| name: ${{ matrix.image_name }}@${{ matrix.target }} | |
| path: "./artifacts/*" | |
| build-docker: | |
| timeout-minutes: 5 | |
| needs: [build-setup, build] | |
| name: Build Docker Image | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| image_name: ${{ fromJson(needs.build-setup.outputs.image_names) }} | |
| env: | |
| PLATFORMS: "${{ join(fromJSON(needs.build-setup.outputs.platforms), ',') }}" | |
| DOCKER_IMAGE: "ghcr.io/getsentry/${{ matrix.image_name }}" | |
| REVISION: "${{ github.event.pull_request.head.sha || github.sha }}" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: "${{ matrix.image_name }}@*" | |
| merge-multiple: true | |
| - name: Build and push to ghcr.io | |
| if: "!github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'" | |
| run: | | |
| docker login --username '${{ github.actor }}' --password '${{ secrets.GITHUB_TOKEN }}' ghcr.io | |
| docker buildx build \ | |
| --platform "${PLATFORMS}" \ | |
| --tag "${DOCKER_IMAGE}:${REVISION}" \ | |
| $( [[ "${GITHUB_REF}" == "refs/heads/master" ]] && printf %s "--tag ${DOCKER_IMAGE}:nightly" ) \ | |
| --file Dockerfile.release \ | |
| --push \ | |
| . | |
| - name: Build and publish docker artifact | |
| if: "github.event.pull_request.head.repo.fork || github.actor == 'dependabot[bot]'" | |
| run: | | |
| docker buildx build \ | |
| --platform "${PLATFORMS}" \ | |
| --tag "${DOCKER_IMAGE}:${REVISION}" \ | |
| --file Dockerfile.release \ | |
| --output type=docker,dest=${{ matrix.image_name }}-docker-image \ | |
| . | |
| - name: Upload docker image | |
| if: "github.event.pull_request.head.repo.fork || github.actor == 'dependabot[bot]'" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 1 | |
| name: ${{ matrix.image_name }}-docker-image | |
| path: "${{ matrix.image_name }}-docker-image" | |
| publish-to-dockerhub: | |
| needs: [build-setup, build-docker] | |
| runs-on: ubuntu-20.04 | |
| name: Publish Relay to DockerHub | |
| strategy: | |
| matrix: | |
| image_name: [relay] # Don't publish relay-pop (for now) | |
| if: ${{ (github.ref_name == 'master') }} | |
| env: | |
| GHCR_DOCKER_IMAGE: "ghcr.io/getsentry/${{ matrix.image_name }}" | |
| DH_DOCKER_IMAGE: "getsentry/${{ matrix.image_name }}" | |
| REVISION: "${{ github.event.pull_request.head.sha || github.sha }}" | |
| steps: | |
| - name: Install cosign | |
| uses: sigstore/cosign-installer@v3.5.0 | |
| - name: Install regctl | |
| uses: regclient/actions/regctl-installer@2dac4eff5925ed07edbfe12d2d11af6304df29a6 | |
| - name: Login to DockerHub | |
| run: docker login --username=sentrybuilder --password ${{ secrets.DOCKER_HUB_RW_TOKEN }} | |
| - name: Copy Image from GHCR to DockerHub | |
| run: | | |
| # We push 3 tags to Dockerhub: | |
| # 1) the full sha of the commit | |
| regctl image copy "${GHCR_DOCKER_IMAGE}:${REVISION}" "${DH_DOCKER_IMAGE}:${REVISION}" | |
| # 2) the short sha | |
| SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-8) | |
| regctl image copy "${GHCR_DOCKER_IMAGE}:${REVISION}" "${DH_DOCKER_IMAGE}:${SHORT_SHA}" | |
| # 3) nightly | |
| regctl image copy "${GHCR_DOCKER_IMAGE}:nightly" "${DH_DOCKER_IMAGE}:nightly" | |
| publish-to-gcr: | |
| timeout-minutes: 5 | |
| needs: [build-setup, build-docker] | |
| name: Publish Relay to GCR | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| image_name: ${{ fromJson(needs.build-setup.outputs.image_names) }} | |
| # required for google auth | |
| permissions: | |
| contents: "read" | |
| id-token: "write" | |
| env: | |
| GHCR_DOCKER_IMAGE: "ghcr.io/getsentry/${{ matrix.image_name }}" | |
| AR_DOCKER_IMAGE: "us-central1-docker.pkg.dev/sentryio/relay/${{ matrix.image_name }}" | |
| REVISION: "${{ github.event.pull_request.head.sha || github.sha }}" | |
| # Skip redundant checks for library releases | |
| # Skip for dependabot and if run on a fork | |
| if: "!startsWith(github.ref, 'refs/heads/release-library/') && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' && needs.build-setup.outputs.full_ci == 'true'" | |
| steps: | |
| - name: Install cosign | |
| uses: sigstore/cosign-installer@v3.5.0 | |
| - name: Google Auth | |
| id: auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool | |
| service_account: gha-gcr-push@sac-prod-sa.iam.gserviceaccount.com | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v2" | |
| with: | |
| # https://github.com/google-github-actions/auth#authenticating-via-workload-identity-federation | |
| # You must use the Cloud SDK version 390.0.0 or later to authenticate the bq and gsutil tools. | |
| version: ">= 390.0.0" | |
| - name: Configure docker | |
| run: | | |
| gcloud auth configure-docker us-central1-docker.pkg.dev | |
| - name: Install regctl | |
| uses: regclient/actions/regctl-installer@2dac4eff5925ed07edbfe12d2d11af6304df29a6 | |
| - name: Copy Image from GHCR to AR | |
| run: regctl image copy "${GHCR_DOCKER_IMAGE}:${REVISION}" "${AR_DOCKER_IMAGE}:${REVISION}" | |
| - name: Copy Nightly from GHCR to AR | |
| if: github.ref == 'refs/heads/master' | |
| run: regctl image copy "${GHCR_DOCKER_IMAGE}:nightly" "${AR_DOCKER_IMAGE}:nightly" | |
| gocd-artifacts: | |
| timeout-minutes: 5 | |
| needs: [build-setup, build-docker] | |
| name: Upload build artifacts to gocd | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| image_name: ${{ fromJson(needs.build-setup.outputs.image_names) }} | |
| # required for google auth | |
| permissions: | |
| contents: "read" | |
| id-token: "write" | |
| env: | |
| GHCR_DOCKER_IMAGE: "ghcr.io/getsentry/${{ matrix.image_name }}" | |
| REVISION: "${{ github.event.pull_request.head.sha || github.sha }}" | |
| if: "!startsWith(github.ref, 'refs/heads/release-library/') && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' && needs.build-setup.outputs.full_ci == 'true'" | |
| steps: | |
| - name: Google Auth | |
| id: auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool | |
| service_account: gha-gcr-push@sac-prod-sa.iam.gserviceaccount.com | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v2" | |
| with: | |
| # https://github.com/google-github-actions/auth#authenticating-via-workload-identity-federation | |
| # You must use the Cloud SDK version 390.0.0 or later to authenticate the bq and gsutil tools. | |
| version: ">= 390.0.0" | |
| - name: Upload gocd deployment assets | |
| run: | | |
| set -euxo pipefail | |
| VERSION="$(docker run --rm "${GHCR_DOCKER_IMAGE}:${REVISION}" --version | cut -d" " -f2)" | |
| echo "${{ matrix.image_name }}@${VERSION}+${REVISION}" > release-name | |
| docker run --rm --entrypoint cat "${GHCR_DOCKER_IMAGE}:${REVISION}" /opt/relay-debug.zip > relay-debug.zip | |
| docker run --rm --entrypoint cat "${GHCR_DOCKER_IMAGE}:${REVISION}" /opt/relay.src.zip > relay.src.zip | |
| docker run --rm --entrypoint tar "${GHCR_DOCKER_IMAGE}:${REVISION}" -cf - /lib/x86_64-linux-gnu > libs.tar | |
| # debugging for mysterious "Couldn't write tracker file" issue: | |
| (env | grep runner) || true | |
| ls -ld \ | |
| /home \ | |
| /home/runner \ | |
| /home/runner/.gsutil \ | |
| /home/runner/.gsutil/tracker-files \ | |
| /home/runner/.gsutil/tracker-files/upload_TRACKER_*.rc.zip__JSON.url \ | |
| || true | |
| gsutil -m cp -L gsutil.log ./libs.tar ./relay-debug.zip ./relay.src.zip ./release-name \ | |
| "gs://dicd-team-devinfra-cd--relay/deployment-assets/${REVISION}/${{ matrix.image_name }}/" || status=$? && status=$? | |
| cat gsutil.log | |
| exit "$status" | |
| test_integration: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| # Skip redundant checks for library releases | |
| if: "!startsWith(github.ref, 'refs/heads/release-library/')" | |
| services: | |
| redis: # https://docs.github.com/en/actions/guides/creating-redis-service-containers | |
| image: redis | |
| ports: | |
| - 6379:6379 | |
| # Kafka + Zookeeper version synced with | |
| # https://github.com/getsentry/sentry/blob/363509c242aff197409207ce4990fb061f3534a3/.github/actions/setup-sentry/action.yml#L174 | |
| zookeeper: | |
| image: confluentinc/cp-zookeeper:4.1.0 | |
| env: | |
| ZOOKEEPER_CLIENT_PORT: 2181 | |
| kafka: | |
| image: confluentinc/cp-kafka:5.1.2 | |
| env: | |
| KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 | |
| KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092 | |
| KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
| KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1 | |
| ports: | |
| - 9092:9092 | |
| steps: | |
| - name: Install libcurl-dev | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libcurl4-openssl-dev | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Rust Toolchain | |
| run: rustup toolchain install stable --profile minimal --no-self-update | |
| - uses: swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ github.job }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - run: make test-integration | |
| env: | |
| PYTEST_N: 6 | |
| RELAY_VERSION_CHAIN: "20.6.0,latest" | |
| sentry-relay-integration-tests: | |
| name: Sentry-Relay Integration Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| needs: build-docker | |
| # Skip redundant checks for library releases | |
| if: "!startsWith(github.ref, 'refs/heads/release-library/')" | |
| steps: | |
| # Checkout Sentry and run integration tests against latest Relay | |
| - name: Checkout Sentry | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: getsentry/sentry | |
| path: sentry | |
| - name: Setup steps | |
| id: setup | |
| run: | | |
| # GITHUB_SHA in pull requests points to the merge commit | |
| RELAY_TEST_IMAGE=ghcr.io/getsentry/relay:${{ github.event.pull_request.head.sha || github.sha }} | |
| echo "We expected GCB to push this image $RELAY_TEST_IMAGE" | |
| echo "relay-test-image=$RELAY_TEST_IMAGE" >> "$GITHUB_OUTPUT" | |
| # We cannot execute actions that are not placed under .github of the main repo | |
| mkdir -p .github/actions | |
| cp -r sentry/.github/actions/setup-sentry .github/actions/ | |
| - name: Setup Sentry | |
| uses: ./.github/actions/setup-sentry | |
| with: | |
| workdir: sentry | |
| snuba: true | |
| kafka: true | |
| symbolicator: true | |
| - name: Download Docker Image | |
| if: "github.event.pull_request.head.repo.fork || github.actor == 'dependabot[bot]'" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: relay-docker-image | |
| - name: Import Docker Image | |
| if: "github.event.pull_request.head.repo.fork || github.actor == 'dependabot[bot]'" | |
| run: docker load -i relay-docker-image | |
| - name: Run Sentry integration tests | |
| working-directory: sentry | |
| env: | |
| RELAY_TEST_IMAGE: ${{ steps.setup.outputs.relay-test-image }} | |
| run: | | |
| echo "Testing against ${RELAY_TEST_IMAGE}" | |
| make test-relay-integration | |
| self-hosted-end-to-end: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| needs: build-docker | |
| # - Skip redundant checks for library releases | |
| # - Skip for dependabot or if it's a fork as the image cannot be uploaded to ghcr since this test attempts to pull | |
| # the image from ghcr | |
| if: "!startsWith(github.ref, 'refs/heads/release-library/') && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'" | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Run Sentry self-hosted e2e CI | |
| uses: getsentry/action-self-hosted-e2e-tests@main | |
| with: | |
| project_name: relay | |
| image_url: ghcr.io/getsentry/relay:${{ github.event.pull_request.head.sha || github.sha }} | |
| docker_repo: getsentry/relay | |
| docker_password: ${{ secrets.DOCKER_HUB_RW_TOKEN }} |