Merge branch 'master' into rachel/json

.github/workflows/docs.yml

@@ -30,7 +30,7 @@ jobs:
  force_orphan: true
 
  - name: Archive Docs
- uses: actions/upload-artifact@v2
+ uses: actions/upload-artifact@v4
  with:
  name: docs
  path: docs/build

CHANGELOG.md

@@ -1,5 +1,36 @@
 # Changelog
 
+## 24.9.0
+
+### Various fixes & improvements
+
+- Update migrations list command to show migrations that no longer exist in the codebase (#6299) by @davidtsuk
+- metric(consumer): Add a metric to track the size of individual spans (#6300) by @ayirr7
+- feat(rpc): Update tags list rpc (#6301) by @Zylphrex
+- feat(eap): add virtual column support (#6292) by @volokluev
+- tweak(eap): Allow more memory usage for eap spans (#6298) by @volokluev
+- ref(doc): add documentation for the ReadinessState enum (#6295) by @viglia
+- feat(eap): Start ingesting data into sample_weight_2 column (#6290) by @colin-sentry
+- Update docker entrypoint to run heaptrack (#6273) by @ayirr7
+- fix(eap): Switch to sampling_weight_2 in entity (#6287) by @colin-sentry
+- bug(query): Run entity validators in composite query pipeline (#6285) by @enochtangg
+- feat(eap): make mapContains work with EAP dataset (#6284) by @colin-sentry
+- feat(job-runner): create a new `snuba jobs` command (#6281) by @xurui-c
+- feat(eap): Shard meta tables by trace ID (#6286) by @colin-sentry
+- fix(eap): Make span_id be returned as a string correctly (#6283) by @colin-sentry
+- feat(job-runner): scaffolding for job manifest testing (#6282) by @onewland
+- bug(admin): Fix invalid query error alerting in snuba admin (#6280) by @enochtangg
+- Fixing Snuba Admin trace UI error. (#6278) by @nachivrn
+- feat(eap): Add a processor that allows you to do mapKeys on attr_str (#6277) by @colin-sentry
+- cleanup(capman): remove legacy table rate limits (#6274) by @volokluev
+- Fixing Snuba Admin trace UI error. (#6276) by @nachivrn
+- hackweek(snuba-admin): MQL query tool (#6235) by @enochtangg
+- feat(eap): Endpoint to get the tags available for a project (#6270) by @colin-sentry
+- feat(sudo): issue slack notifications when sudo mode is used (#6271) by @volokluev
+- chore(eap): Add entities and storages for EAP span meta tables (#6269) by @colin-sentry
+
+_Plus 60 more_
+
 ## 24.8.0
 
 ### Various fixes & improvements

CODEOWNERS

@@ -16,6 +16,7 @@
 
 # EAP-related
 /snuba/web/rpc @getsentry/events-analytics-platform @getsentry/owners-snuba
+/tests/web/rpc @getsentry/events-analytics-platform @getsentry/owners-snuba
 /snuba/snuba_migrations/events_analytics_platform @getsentry/events-analytics-platform @getsentry/owners-snuba
 /rust_snuba/src/processors/eap_spans.rs @getsentry/events-analytics-platform
 

Dockerfile

@@ -30,6 +30,8 @@ RUN set -ex; \
  runtimeDeps=' \
  curl \
  libjemalloc2 \
+ gdb \
+ heaptrack \
  '; \
  apt-get update; \
  apt-get install -y $buildDeps $runtimeDeps --no-install-recommends; \

docker_entrypoint.sh

@@ -18,4 +18,8 @@ else
  printf "\n${help_result}"
 fi
 
+if [ -n "${ENABLE_HEAPTRACK:-}" ]; then
+ set -- heaptrack "$@"
+fi
+
 exec "$@"

docs/source/conf.py

@@ -7,7 +7,7 @@
 copyright = "2021, Sentry Team and Contributors"
 author = "Sentry Team and Contributors"
 
-release = "24.9.0.dev0"
+release = "24.10.0.dev0"
 
 
 # -- General configuration ---------------------------------------------------

requirements.txt

@@ -26,7 +26,7 @@ pytest-watch==4.2.0
 python-dateutil==2.8.2
 python-rapidjson==1.8
 redis==4.3.4
-sentry-arroyo==2.17.1
+sentry-arroyo==2.17.6
 sentry-kafka-schemas==0.1.106
 sentry-redis-tools==0.3.0
 sentry-relay==0.8.44
@@ -45,4 +45,4 @@ sqlparse==0.4.2
 google-api-python-client==2.88.0
 sentry-usage-accountant==0.0.10
 freezegun==1.2.2
-sentry-protos==0.1.16
+sentry-protos==0.1.21

rust_snuba/bin/python_processor_infinite.rs

@@ -20,9 +20,9 @@ fn main() {
  let output2 = output.clone();
 
  let step = RunTask::new(
- move |_| {
+ move |message| {
  output2.fetch_add(1, Ordering::Relaxed);
- Ok(())
+ Ok(message)
  },
  step,
  );

rust_snuba/src/processors/eap_spans.rs

@@ -52,7 +52,8 @@ struct EAPSpan {
  name: String, //aka description
 
  sampling_factor: f64,
- sampling_weight: f64,
+ sampling_weight: f64, //remove eventually
+ sampling_weight_2: u64,
  sign: u8, //1 for additions, -1 for deletions - for this worker it should be 1
 
  #(
@@ -101,7 +102,8 @@ impl From<FromSpanMessage> for EAPSpan {
  retention_days: from.retention_days,
  name: from.description.unwrap_or_default(),
 
- sampling_weight: 1.,
+ sampling_weight: 1., //remove eventually
+ sampling_weight_2: 1,
  sampling_factor: 1.,
  sign: 1,
 
@@ -153,6 +155,7 @@ impl From<FromSpanMessage> for EAPSpan {
  if k == "client_sample_rate" && v.value != 0.0 {
  res.sampling_factor = v.value;
  res.sampling_weight = 1.0 / v.value;
+ res.sampling_weight_2 = (1.0 / v.value) as u64;
  } else {
  insert_num(k.clone(), v.value);
  }
@@ -217,6 +220,9 @@ mod tests {
  "measurements": {
  "num_of_spans": {
  "value": 50.0
+ },
+ "client_sample_rate": {
+ "value": 0.01
  }
  },
  "organization_id": 1,

rust_snuba/src/processors/snapshots/rust_snuba__processors__eap_spans__tests__serialization.snap

@@ -19,8 +19,9 @@ expression: span
  "exclusive_time_ms": 0.228,
  "retention_days": 90,
  "name": "/api/0/relays/projectconfigs/",
- "sampling_factor": 1.0,
- "sampling_weight": 1.0,
+ "sampling_factor": 0.01,
+ "sampling_weight": 100.0,
+ "sampling_weight_2": 100,
  "sign": 1,
  "attr_str_0": {
  "relay_protocol_version": "3",

rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-spans-EAPSpansMessageProcessor-snuba-spans__1__basic_span.json.snap

@@ -52,6 +52,7 @@ expression: snapshot_payload
  "retention_days": 90,
  "sampling_factor": 1.0,
  "sampling_weight": 1.0,
+ "sampling_weight_2": 1,
  "segment_id": 16045690984833335023,
  "segment_name": "/organizations/:orgId/issues/",
  "service": "1",

rust_snuba/src/processors/spans.rs

@@ -3,6 +3,7 @@ use std::str::FromStr;
 
 use anyhow::Context;
 use chrono::DateTime;
+use rust_arroyo::timer;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
@@ -22,6 +23,8 @@ pub fn process_message(
  let payload_bytes = payload.payload().context("Expected payload")?;
  let msg: FromSpanMessage = serde_json::from_slice(payload_bytes)?;
 
+ timer!("spans.messages.size", payload_bytes.len() as f64);
+
  let origin_timestamp = DateTime::from_timestamp(msg.received as i64, 0);
  let mut span: Span = msg.try_into()?;
 

setup.py

@@ -2,7 +2,7 @@
 
 from setuptools import find_packages, setup
 
-VERSION = "24.9.0.dev0"
+VERSION = "24.10.0.dev0"
 
 
 def get_requirements() -> Sequence[str]:

snuba/admin/clickhouse/migration_checks.py

@@ -100,7 +100,7 @@ def __init__(
  ).get_migrations()
 
  migration_statuses = {}
- for migration_id, status, _ in migrations:
+ for migration_id, status, _, _ in migrations:
  migration_statuses[migration_id] = {
  "migration_id": migration_id,
  "status": status,

snuba/admin/notifications/slack/utils.py

@@ -1,3 +1,4 @@
+import os
 from typing import Any, Dict, List, Optional, Union
 
 from snuba import settings
@@ -95,12 +96,13 @@ def build_context(
  user: str, timestamp: str, action: AuditLogAction
 ) -> Dict[str, Union[str, List[Dict[str, str]]]]:
  url = f"{settings.ADMIN_URL}/#auditlog"
+ environ = os.environ.get("SENTRY_ENVIRONMENT") or "unknown environment"
  return {
  "type": "context",
  "elements": [
  {
  "type": "mrkdwn",
- "text": f"{action.value} at *<{url}|{timestamp}>* by *<{user}>*",
+ "text": f"{action.value} at *<{url}|{timestamp}>* by *<{user}>* in *<{environ}>*",
  }
  ],
  }

snuba/admin/views.py

@@ -208,7 +208,7 @@ def migrations_groups_list(group: str) -> Response:
  "status": status.value,
  "blocking": blocking,
  }
- for migration_id, status, blocking in runner_group_migrations
+ for migration_id, status, blocking, _ in runner_group_migrations
  ]
  ),
  200,

snuba/cli/migrations.py

@@ -37,10 +37,10 @@ def list() -> None:
  setup_logging()
  check_clickhouse_connections(CLUSTERS)
  runner = Runner()
- for group, group_migrations in runner.show_all():
+ for group, group_migrations in runner.show_all(include_nonexistent=True):
  readiness_state = get_group_readiness_state(group)
  click.echo(f"{group.value} (readiness_state: {readiness_state.value})")
- for migration_id, status, blocking in group_migrations:
+ for migration_id, status, blocking, existing in group_migrations:
  symbol = {
  Status.COMPLETED: "X",
  Status.NOT_STARTED: " ",
@@ -53,7 +53,11 @@ def list() -> None:
  if status != Status.COMPLETED and blocking:
  blocking_text = " (blocking)"
 
- click.echo(f"[{symbol}] {migration_id}{in_progress_text}{blocking_text}")
+ existing_text = "" if existing else " (this migration no longer exists)"
+
+ click.echo(
+ f"[{symbol}] {migration_id}{in_progress_text}{blocking_text}{existing_text}"
+ )
 
  click.echo()
 

snuba/clickhouse/http.py

@@ -353,7 +353,9 @@ def write(self, values: Iterable[bytes]) -> None:
  batch.append(value)
 
  batch.close()
- batch_join_timeout = state.get_config("http_batch_join_timeout", 10)
+ batch_join_timeout = state.get_config(
+ "http_batch_join_timeout", settings.BATCH_JOIN_TIMEOUT
+ )
  # IMPORTANT: Please read the docstring of this method if you ever decide to remove the
  # timeout argument from the join method.
  batch.join(timeout=batch_join_timeout)

snuba/datasets/configuration/events_analytics_platform/entities/eap_spans.yaml

@@ -23,6 +23,7 @@ schema:
  { name: name, type: String },
  { name: sampling_factor, type: Float, args: { size: 64 } },
  { name: sampling_weight, type: Float, args: { size: 64 } },
+ { name: sampling_weight_2, type: UInt, args: { size: 64 } },
  { name: sign, type: Int, args: { size: 8 } },
  { name: attr_str, type: Map, args: { key: { type: String }, value: { type: String } } },
  { name: attr_num, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } },
@@ -39,6 +40,7 @@ storages:
  from_col_name: timestamp
  to_table_name: null
  to_col_name: _sort_timestamp
+
  subscriptables:
  - mapper: SubscriptableHashBucketMapper
  args:

snuba/datasets/configuration/events_analytics_platform/storages/eap_spans.yaml

@@ -29,6 +29,7 @@ schema:
  { name: name, type: String },
  { name: sampling_factor, type: Float, args: { size: 64 } },
  { name: sampling_weight, type: Float, args: { size: 64 } },
+ { name: sampling_weight_2, type: UInt, args: { size: 64 } },
  { name: sign, type: Int, args: { size: 8 } },
  { name: attr_str_0, type: Map, args: { key: { type: String }, value: { type: String } } },
  { name: attr_str_1, type: Map, args: { key: { type: String }, value: { type: String } } },
@@ -113,6 +114,12 @@ query_processors:
  prewhere_candidates:
  [span_id, trace_id, segment_name]
  - processor: TupleUnaliaser
+ - processor: ClickhouseSettingsOverride
+ args:
+ settings:
+ max_memory_usage: 5000000000
+ max_rows_to_group_by: 1000000
+ group_by_overflow_mode: any
 
 mandatory_condition_checkers:
  - condition: OrgIdEnforcer

snuba/datasets/readiness_state.py

@@ -4,6 +4,26 @@
 
 
 class ReadinessState(Enum):
+ """
+ Readiness states are essentially feature flags for snuba datasets.
+ The readiness state defines whether or not a dataset is made available
+ in specific sentry environments.
+ Currently, sentry environments include the following:
+ * local/CI
+ * SaaS
+ * S4S
+ * Self-Hosted
+ * Single-Tenant
+
+ The following is a list of readiness states and the environments
+ they map to:
+ * limited -> local/CI
+ * experimental -> local/CI, S4S
+ * partial -> local/CI, SaaS, S4S
+ * deprecate -> local/CI, Self-Hosted
+ * complete -> local/CI, SaaS, S4S, Self-Hosted, Single-Tenant
+ """
+
  LIMITED = "limited"
  DEPRECATE = "deprecate"
  PARTIAL = "partial"