merge conflicts

.github/workflows/admin-sourcemaps.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         name: Checkout code
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - uses: actions/setup-node@v3

.github/workflows/ci.yml

@@ -37,10 +37,10 @@ jobs:
         with:
           app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }}
           private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }}
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
           key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
@@ -76,7 +76,7 @@ jobs:
       # If working tree is dirty, commit and update if we have a token
       - name: Apply any pre-commit fixed files
         if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always()
-        uses: getsentry/action-github-commit@v1.0.0
+        uses: getsentry/action-github-commit@v2.1.0
         with:
           github-token: ${{ steps.token.outputs.token }}
 
@@ -98,7 +98,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         name: Checkout code
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
       - name: Install dependencies
@@ -116,7 +116,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         name: Checkout code
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
       - name: Install dependencies
@@ -153,7 +153,7 @@ jobs:
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
 
       - name: Build snuba docker image for CI
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         if: github.repository_owner == 'getsentry'
         with:
           context: .

.github/workflows/codeql-analysis.yml

@@ -37,7 +37,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           config-file: ./.github/codeql/codeql-config.yml
           languages: ${{ matrix.language }}
@@ -49,7 +49,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -63,4 +63,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/ddl-changes.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           clean: false
           fetch-depth: 200
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
       - name: Install dependencies

.github/workflows/docs-pr.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.8'
       - name: Generate config schema docs

.github/workflows/docs.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.8'
       - name: Generate config schema docs
@@ -22,7 +22,7 @@ jobs:
       - name: Build docs
         run: |
           make snubadocs
-      - uses: peaceiris/actions-gh-pages@v3.7.3
+      - uses: peaceiris/actions-gh-pages@v4.0.0
         name: Publish to GitHub Pages
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}

.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "ms-python.black-formatter",
+        "ms-python.mypy-type-checker",
+        "ms-python.flake8"
+    ]
+}

.vscode/settings.json

@@ -26,21 +26,25 @@
   },
 
   "[javascript]": {
-    "editor.formatOnSave": true
+    "editor.formatOnSave": true,
+    "editor.tabSize": 2
   },
 
   "[javascriptreact]": {
-    "editor.formatOnSave": true
+    "editor.formatOnSave": true,
+    "editor.tabSize": 2
   },
 
   "[typescript]": {
     "editor.formatOnSave": true,
-    "editor.defaultFormatter": "esbenp.prettier-vscode"
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.tabSize": 2
   },
 
   "[typescriptreact]": {
     "editor.formatOnSave": true,
-    "editor.defaultFormatter": "vscode.typescript-language-features"
+    "editor.defaultFormatter": "vscode.typescript-language-features",
+    "editor.tabSize": 2
   },
 
   "[less]": {
@@ -56,11 +60,7 @@
     "editor.formatOnSave": true
   },
 
-  "python.linting.mypyEnabled": true,
-  "python.linting.flake8Enabled": true,
-  "python.formatting.provider": "black",
-  // https://github.com/DonJayamanne/pythonVSCode/issues/992
-  "python.pythonPath": ".venv/bin/python",
+  "python.defaultInterpreterPath": ".venv/bin/python",
   // test discovery is sluggish and the UI around running
   // tests is often in your way and misclicked
   "python.testing.pytestEnabled": true,

Dockerfile

@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION=3.11.8
 
-FROM python:${PYTHON_VERSION}-slim-bookworm as build_base
+FROM python:${PYTHON_VERSION}-slim-bookworm AS build_base
 WORKDIR /usr/src/snuba
 
 ENV PIP_NO_CACHE_DIR=off \
@@ -150,7 +150,7 @@ EXPOSE 1218 1219
 ENTRYPOINT [ "./docker_entrypoint.sh" ]
 CMD [ "api" ]
 
-FROM application_base as application
+FROM application_base AS application
 USER 0
 RUN set -ex; \
     apt-get purge -y --auto-remove $(cat /tmp/build-deps.txt); \

requirements-test.txt

@@ -7,7 +7,7 @@ mypy==1.1.1
 types-python-dateutil==2.8.19
 types-python-jose==3.3.0
 types-pyyaml==6.0.11
-types-requests==2.28.10
+types-requests==2.32.0.20240907
 types-setuptools==65.3.0
 types-simplejson==3.17.7
 types-google-cloud-ndb==2.2.0

requirements.txt

@@ -7,7 +7,7 @@ clickhouse-driver==0.2.6
 confluent-kafka==2.3.0
 datadog==0.21.0
 devservices==0.0.4
-flake8==5.0.4
+flake8==7.0.0
 Flask==2.2.5
 google-cloud-storage==2.18.0
 googleapis-common-protos==1.63.2
@@ -28,22 +28,22 @@ python-dateutil==2.8.2
 python-rapidjson==1.8
 redis==4.5.4
 sentry-arroyo==2.17.6
-sentry-kafka-schemas==0.1.115
+sentry-kafka-schemas==0.1.117
 sentry-redis-tools==0.3.0
-sentry-relay==0.8.44
+sentry-relay==0.9.2
 sentry-sdk==2.8.0
 simplejson==3.17.6
 snuba-sdk==3.0.39
 structlog==22.3.0
 structlog-sentry==2.0.0
 sql-metadata==2.11.0
 typing-extensions==4.8.0
-urllib3==1.26.12
+urllib3==2.2.3
 pyuwsgi==2.0.23
 Werkzeug==3.0.5
 PyYAML==6.0
 sqlparse==0.5.0
 google-api-python-client==2.88.0
 sentry-usage-accountant==0.0.10
 freezegun==1.2.2
-sentry-protos==0.1.32
+sentry-protos==0.1.34

rust_snuba/src/mutations/factory.rs

@@ -102,7 +102,7 @@ impl ProcessingStrategyFactory<KafkaPayload> for MutConsumerStrategyFactory {
         );
 
         let mut synchronizer = Synchronizer {
-            min_delay: TimeDelta::hours(48),
+            min_delay: TimeDelta::hours(1),
         };
 
         let next_step = RunTask::new(move |m| synchronizer.process_message(m), next_step);

rust_snuba/src/processors/eap_spans.rs

@@ -29,9 +29,16 @@ pub fn process_message(
     _metadata: KafkaMessageMetadata,
     config: &ProcessorConfig,
 ) -> anyhow::Result<InsertBatch> {
+    if let Some(headers) = payload.headers() {
+        if let Some(ingest_in_eap) = headers.get("ingest_in_eap") {
+            if ingest_in_eap == b"false" {
+                return Ok(InsertBatch::skip());
+            }
+        }
+    }
+
     let payload_bytes = payload.payload().context("Expected payload")?;
     let msg: FromSpanMessage = serde_json::from_slice(payload_bytes)?;
-
     let origin_timestamp = DateTime::from_timestamp(msg.received as i64, 0);
     let mut span: EAPSpan = msg.try_into()?;
 

snuba/admin/clickhouse/common.py

@@ -58,7 +58,12 @@ def _validate_node(
     if not is_valid_node(clickhouse_host, clickhouse_port, cluster, storage_name):
         raise InvalidNodeError(
             f"host {clickhouse_host} and port {clickhouse_port} are not valid",
-            extra_data={"host": clickhouse_host, "port": clickhouse_port},
+            extra_data={
+                "host": clickhouse_host,
+                "port": clickhouse_port,
+                "query_host": cluster.get_query_node().host_name,
+                "query_port": cluster.get_query_node().port,
+            },
         )