Merge branch 'master' into david/feat/prod-query-api

docs-requirements.txt

@@ -1,5 +1,5 @@
 jsonschema2md==0.4.0
 fastjsonschema==2.16.2
-sentry-sdk==1.18.0
+sentry-sdk==1.26.0
 myst-parser==0.18.0
 sphinx==5.1.1

gocd/pipelines/snuba.yaml

@@ -112,6 +112,7 @@ pipelines:
  --container-name="transactions-subscriptions-executor" \
  --container-name="transactions-subscriptions-scheduler" \
  --container-name="spans-consumer" \
+ --container-name="dlq-consumer" \
  && /devinfra/scripts/k8s/k8s-deploy.py \
  --label-selector="service=snuba,is_canary=true" \
  --image="us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \

requirements.txt

@@ -24,10 +24,10 @@ python-rapidjson==1.8
 pytz==2022.2.1
 redis==4.3.4
 sentry-arroyo==2.13.0
-sentry-kafka-schemas==0.1.12
+sentry-kafka-schemas==0.1.14
 sentry-redis-tools==0.1.6
 sentry-relay==0.8.21
-sentry-sdk==1.18.0
+sentry-sdk==1.26.0
 simplejson==3.17.6
 structlog==22.3.0
 structlog-sentry==2.0.0

snuba/admin/auth.py

@@ -3,6 +3,7 @@
 import json
 from typing import Sequence
 
+import rapidjson
 import structlog
 from flask import request
 
@@ -11,9 +12,12 @@
 from snuba.admin.google import CloudIdentityAPI
 from snuba.admin.jwt import validate_assertion
 from snuba.admin.user import AdminUser
+from snuba.redis import RedisClientKey, get_redis_client
 
 USER_HEADER_KEY = "X-Goog-Authenticated-User-Email"
 
+redis_client = get_redis_client(RedisClientKey.ADMIN_AUTH)
+
 logger = structlog.get_logger().bind(module=__name__)
 
 
@@ -41,7 +45,7 @@ def _is_member_of_group(user: AdminUser, group: str) -> bool:
  return google_api.check_group_membership(group_email=group, member=user.email)
 
 
-def get_iam_roles_from_file(user: AdminUser) -> Sequence[str]:
+def get_iam_roles_from_user(user: AdminUser) -> Sequence[str]:
  iam_roles = []
  try:
  with open(settings.ADMIN_IAM_POLICY_FILE, "r") as policy_file:
@@ -65,10 +69,38 @@ def get_iam_roles_from_file(user: AdminUser) -> Sequence[str]:
  return iam_roles
 
 
+def get_cached_iam_roles(user: AdminUser) -> Sequence[str]:
+ iam_roles_str = redis_client.get(f"roles-{user.email}")
+ if not iam_roles_str:
+ return []
+
+ iam_roles = rapidjson.loads(iam_roles_str)
+ if isinstance(iam_roles, list):
+ return iam_roles
+
+ return []
+
+
 def _set_roles(user: AdminUser) -> AdminUser:
  # todo: depending on provider convert user email
  # to subset of DEFAULT_ROLES based on IAM roles
- iam_roles = get_iam_roles_from_file(user)
+ iam_roles: Sequence[str] = []
+ try:
+ iam_roles = get_cached_iam_roles(user)
+ except Exception as e:
+ logger.exception("Failed to load roles from cache", exception=e)
+
+ if not iam_roles:
+ iam_roles = get_iam_roles_from_user(user)
+ try:
+ redis_client.set(
+ f"roles-{user.email}",
+ rapidjson.dumps(iam_roles),
+ ex=settings.ADMIN_ROLES_REDIS_TTL,
+ )
+ except Exception as e:
+ logger.exception(e)
+
  user.roles = [*[ROLES[role] for role in iam_roles if role in ROLES], *DEFAULT_ROLES]
  return user
 

snuba/admin/package.json

@@ -7,9 +7,9 @@
  "test": "jest"
  },
  "dependencies": {
- "@sentry/react": "^7.53.1",
+ "@sentry/react": "^7.56.0",
  "@types/react": "^18.0.20",
- "@types/react-dom": "^18.0.6",
+ "@types/react-dom": "^18.2.6",
  "jest-dom": "^4.0.0",
  "react": "^18.2.0",
  "react-dom": "^18.2.0",
@@ -27,7 +27,7 @@
  "jest-environment-jsdom": "^29.5.0",
  "react-bootstrap": "^2.7.4",
  "ts-jest": "^29.0.5",
- "webpack": "^5.74.0",
+ "webpack": "^5.88.0",
  "webpack-cli": "^4.10.0"
  },
  "volta": {