diff --git a/docs-requirements.txt b/docs-requirements.txt index 071c61f729..3ac18c8f7a 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,5 +1,5 @@ jsonschema2md==0.4.0 fastjsonschema==2.16.2 -sentry-sdk==1.18.0 +sentry-sdk==1.26.0 myst-parser==0.18.0 sphinx==5.1.1 diff --git a/gocd/pipelines/snuba.yaml b/gocd/pipelines/snuba.yaml index 569322fab4..df54e4caac 100644 --- a/gocd/pipelines/snuba.yaml +++ b/gocd/pipelines/snuba.yaml @@ -112,6 +112,7 @@ pipelines: --container-name="transactions-subscriptions-executor" \ --container-name="transactions-subscriptions-scheduler" \ --container-name="spans-consumer" \ + --container-name="dlq-consumer" \ && /devinfra/scripts/k8s/k8s-deploy.py \ --label-selector="service=snuba,is_canary=true" \ --image="us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ diff --git a/requirements.txt b/requirements.txt index 2de9a9ba9e..e3226d7262 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,10 +24,10 @@ python-rapidjson==1.8 pytz==2022.2.1 redis==4.3.4 sentry-arroyo==2.13.0 -sentry-kafka-schemas==0.1.12 +sentry-kafka-schemas==0.1.14 sentry-redis-tools==0.1.6 sentry-relay==0.8.21 -sentry-sdk==1.18.0 +sentry-sdk==1.26.0 simplejson==3.17.6 structlog==22.3.0 structlog-sentry==2.0.0 diff --git a/rust_snuba/Cargo.lock b/rust_snuba/Cargo.lock index b0f58aaeb6..3752097c58 100644 --- a/rust_snuba/Cargo.lock +++ b/rust_snuba/Cargo.lock @@ -978,9 +978,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.48" +version = "0.10.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2" +checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -1010,11 +1010,10 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.83" +version = "0.9.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" +checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" dependencies = [ - "autocfg", "cc", "libc", "pkg-config", diff --git a/snuba/admin/auth.py b/snuba/admin/auth.py index 92092888c1..5bb5bb3df6 100644 --- a/snuba/admin/auth.py +++ b/snuba/admin/auth.py @@ -3,6 +3,7 @@ import json from typing import Sequence +import rapidjson import structlog from flask import request @@ -11,9 +12,12 @@ from snuba.admin.google import CloudIdentityAPI from snuba.admin.jwt import validate_assertion from snuba.admin.user import AdminUser +from snuba.redis import RedisClientKey, get_redis_client USER_HEADER_KEY = "X-Goog-Authenticated-User-Email" +redis_client = get_redis_client(RedisClientKey.ADMIN_AUTH) + logger = structlog.get_logger().bind(module=__name__) @@ -41,7 +45,7 @@ def _is_member_of_group(user: AdminUser, group: str) -> bool: return google_api.check_group_membership(group_email=group, member=user.email) -def get_iam_roles_from_file(user: AdminUser) -> Sequence[str]: +def get_iam_roles_from_user(user: AdminUser) -> Sequence[str]: iam_roles = [] try: with open(settings.ADMIN_IAM_POLICY_FILE, "r") as policy_file: @@ -65,10 +69,38 @@ def get_iam_roles_from_file(user: AdminUser) -> Sequence[str]: return iam_roles +def get_cached_iam_roles(user: AdminUser) -> Sequence[str]: + iam_roles_str = redis_client.get(f"roles-{user.email}") + if not iam_roles_str: + return [] + + iam_roles = rapidjson.loads(iam_roles_str) + if isinstance(iam_roles, list): + return iam_roles + + return [] + + def _set_roles(user: AdminUser) -> AdminUser: # todo: depending on provider convert user email # to subset of DEFAULT_ROLES based on IAM roles - iam_roles = get_iam_roles_from_file(user) + iam_roles: Sequence[str] = [] + try: + iam_roles = get_cached_iam_roles(user) + except Exception as e: + logger.exception("Failed to load roles from cache", exception=e) + + if not iam_roles: + iam_roles = get_iam_roles_from_user(user) + try: + redis_client.set( + f"roles-{user.email}", + rapidjson.dumps(iam_roles), + ex=settings.ADMIN_ROLES_REDIS_TTL, + ) + except Exception as e: + logger.exception(e) + user.roles = [*[ROLES[role] for role in iam_roles if role in ROLES], *DEFAULT_ROLES] return user diff --git a/snuba/admin/package.json b/snuba/admin/package.json index 000e2aecaf..60fea6f1b2 100644 --- a/snuba/admin/package.json +++ b/snuba/admin/package.json @@ -7,9 +7,9 @@ "test": "jest" }, "dependencies": { - "@sentry/react": "^7.53.1", + "@sentry/react": "^7.56.0", "@types/react": "^18.0.20", - "@types/react-dom": "^18.0.6", + "@types/react-dom": "^18.2.6", "jest-dom": "^4.0.0", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -27,7 +27,7 @@ "jest-environment-jsdom": "^29.5.0", "react-bootstrap": "^2.7.4", "ts-jest": "^29.0.5", - "webpack": "^5.74.0", + "webpack": "^5.88.0", "webpack-cli": "^4.10.0" }, "volta": { diff --git a/snuba/admin/yarn.lock b/snuba/admin/yarn.lock index 6c1bf66568..b073d750cb 100644 --- a/snuba/admin/yarn.lock +++ b/snuba/admin/yarn.lock @@ -601,68 +601,68 @@ uncontrollable "^7.2.1" warning "^4.0.3" -"@sentry-internal/tracing@7.53.1": - version "7.53.1" - resolved "https://registry.yarnpkg.com/@sentry-internal/tracing/-/tracing-7.53.1.tgz#85517ba93ee721424c865706f7ff4eaab1569e6d" - integrity sha512-a4H4rvVdz0XDGgNfRqc7zg6rMt2P1P05xBmgfIfztYy94Vciw1QMdboNiT7einr8ra8wogdEaK4Pe2AzYAPBJQ== - dependencies: - "@sentry/core" "7.53.1" - "@sentry/types" "7.53.1" - "@sentry/utils" "7.53.1" +"@sentry-internal/tracing@7.56.0": + version "7.56.0" + resolved "https://registry.yarnpkg.com/@sentry-internal/tracing/-/tracing-7.56.0.tgz#ba709258f2f0f3d8a36f9740403088b39212b843" + integrity sha512-OKI4Pz/O13gng8hT9rNc+gRV3+P7nnk1HnHlV8fgaQydS6DsRxoDL1sHa42tZGbh7K9jqNAP3TC6VjBOsr2tXA== + dependencies: + "@sentry/core" "7.56.0" + "@sentry/types" "7.56.0" + "@sentry/utils" "7.56.0" tslib "^1.9.3" -"@sentry/browser@7.53.1": - version "7.53.1" - resolved "https://registry.yarnpkg.com/@sentry/browser/-/browser-7.53.1.tgz#1efd94cd9a56360fc401b99043eeaaac3da2e70e" - integrity sha512-1zas2R6riJaj0k7FoeieCW0SuC7UyKaBGA6jEG2LsgIqyD7IDOlF3BPZ4Yt08GFav0ImpyhGn5Vbrq5JLbeQdw== +"@sentry/browser@7.56.0": + version "7.56.0" + resolved "https://registry.yarnpkg.com/@sentry/browser/-/browser-7.56.0.tgz#6bf3ff21bc2e9b66a72ea0c7dcc3572fdeb3bd8f" + integrity sha512-qpyyw+NM/psbNAYMlTCCdWwxHHcogppEQ+Q40jGE4sKP2VRIjjyteJkUcaEMoGpbJXx9puzTWbpzqlQ8r15Now== dependencies: - "@sentry-internal/tracing" "7.53.1" - "@sentry/core" "7.53.1" - "@sentry/replay" "7.53.1" - "@sentry/types" "7.53.1" - "@sentry/utils" "7.53.1" + "@sentry-internal/tracing" "7.56.0" + "@sentry/core" "7.56.0" + "@sentry/replay" "7.56.0" + "@sentry/types" "7.56.0" + "@sentry/utils" "7.56.0" tslib "^1.9.3" -"@sentry/core@7.53.1": - version "7.53.1" - resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.53.1.tgz#c091a9d7fd010f8a2cb1dd71d949a8e453e35d4c" - integrity sha512-DAH8IJNORJJ7kQLqsZuhMkN6cwJjXzFuuUoZor7IIDHIHjtl51W+2F3Stg3+I3ZoKDfJfUNKqhipk2WZjG0FBg== +"@sentry/core@7.56.0": + version "7.56.0" + resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.56.0.tgz#f4253e0d61f55444180a63e5278b62e57303f7cf" + integrity sha512-Nuyyfh09Yz27kPo74fXHlrdmZeK6zrlJVtxQ6LkwuoaTBcNcesNXVaOtr6gjvUGUmsfriVPP3Jero5LXufV7GQ== dependencies: - "@sentry/types" "7.53.1" - "@sentry/utils" "7.53.1" + "@sentry/types" "7.56.0" + "@sentry/utils" "7.56.0" tslib "^1.9.3" -"@sentry/react@^7.53.1": - version "7.53.1" - resolved "https://registry.yarnpkg.com/@sentry/react/-/react-7.53.1.tgz#885f743b4b48639a4d415ee401632f02e59dbbbb" - integrity sha512-eEOY/peBepSD/nhPn4SU77aYdjQfAI1svOqpG4sbpjaGZU1P6L7+IIGmip8l2T68oPEeKDaiH9Qy/3uxu55B/Q== +"@sentry/react@^7.56.0": + version "7.56.0" + resolved "https://registry.yarnpkg.com/@sentry/react/-/react-7.56.0.tgz#7e2e9363a76c7d67854bdb179142a2f7f910d476" + integrity sha512-dRnkZwspef5aEHV/eiLS/mhomFCXItylU8s78fzAn5QMTDGHmu5Pnhl5dxh/zbPUcdXqFA6GwJVucV4gzsNEJw== dependencies: - "@sentry/browser" "7.53.1" - "@sentry/types" "7.53.1" - "@sentry/utils" "7.53.1" + "@sentry/browser" "7.56.0" + "@sentry/types" "7.56.0" + "@sentry/utils" "7.56.0" hoist-non-react-statics "^3.3.2" tslib "^1.9.3" -"@sentry/replay@7.53.1": - version "7.53.1" - resolved "https://registry.yarnpkg.com/@sentry/replay/-/replay-7.53.1.tgz#543272827d3ca034c62b0a822503d0a57c9229e7" - integrity sha512-5He5JLJiYLeWtXHC53z2ZzfbgAedafbHNZVS4+MBCOtydCk7cnuyJ0gGV6Rfxej/lZSNXZxOdW7HeMhzBtZCxw== +"@sentry/replay@7.56.0": + version "7.56.0" + resolved "https://registry.yarnpkg.com/@sentry/replay/-/replay-7.56.0.tgz#8a49dcb45e9ea83bf905cec0d9b42fed4b8085bd" + integrity sha512-bvjiJK1+SM/paLapuL+nEJ8CIF1bJqi0nnFyxUIi2L5L6yb2uMwfyT3IQ+kz0cXJsLdb3HN4WMphVjyiU9pFdg== dependencies: - "@sentry/core" "7.53.1" - "@sentry/types" "7.53.1" - "@sentry/utils" "7.53.1" + "@sentry/core" "7.56.0" + "@sentry/types" "7.56.0" + "@sentry/utils" "7.56.0" -"@sentry/types@7.53.1": - version "7.53.1" - resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.53.1.tgz#3eefbad851f2d0deff67285d7e976d23d7d06a41" - integrity sha512-/ijchRIu+jz3+j/zY+7KRPfLSCY14fTx5xujjbOdmEKjmIHQmwPBdszcQm40uwofrR8taV4hbt5MFN+WnjCkCw== +"@sentry/types@7.56.0": + version "7.56.0" + resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.56.0.tgz#9042a099cf9e8816d081886d24b88082a5d9f87a" + integrity sha512-5WjhVOQm75ItOytOx2jTx+5yw8/qJ316+g1Di8dS9+kgIi1zniqdMcX00C2yYe3FMUgFB49PegCUYulm9Evapw== -"@sentry/utils@7.53.1": - version "7.53.1" - resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.53.1.tgz#b1f9f1dd4de7127287ad5027c2bd1133c5590486" - integrity sha512-DKJA1LSUOEv4KOR828MzVuLh+drjeAgzyKgN063OEKmnirgjgRgNNS8wUgwpG0Tn2k6ANZGCwrdfzPeSBxshKg== +"@sentry/utils@7.56.0": + version "7.56.0" + resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.56.0.tgz#e60e4935d17b2197584abf6ce61b522ad055352c" + integrity sha512-wgeX7bufxc//TjjSIE+gCMm8hVId7Jzvc+f441bYrWnNZBuzPIDW2BummCcPrKzSYe5GeYZDTZGV8YZGMLGBjw== dependencies: - "@sentry/types" "7.53.1" + "@sentry/types" "7.56.0" tslib "^1.9.3" "@sinclair/typebox@^0.25.16": @@ -860,17 +860,10 @@ resolved "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.4.tgz" integrity sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ== -"@types/react-dom@^18.0.0": - version "18.0.11" - resolved "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.0.11.tgz" - integrity sha512-O38bPbI2CWtgw/OoQoY+BRelw7uysmXbWvw3nLWO21H1HSh+GOlqPuXshJfjmpNlKiiSDG9cc1JZAaMmVdcTlw== - dependencies: - "@types/react" "*" - -"@types/react-dom@^18.0.6": - version "18.0.6" - resolved "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.0.6.tgz" - integrity sha512-/5OFZgfIPSwy+YuIBP/FgJnQnsxhZhjjrnxudMddeblOouIodEQ75X14Rr4wGSG/bknL+Omy9iWlLo1u/9GzAA== +"@types/react-dom@^18.0.0", "@types/react-dom@^18.2.6": + version "18.2.6" + resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-18.2.6.tgz#ad621fa71a8db29af7c31b41b2ea3d8a6f4144d1" + integrity sha512-2et4PDvg6PVCyS7fuTc4gPoksV58bW0RwSxWKcPRcHZf0PRUGq03TKcD/rUHe3azfV6/5/biUBJw+HhCQjaP0A== dependencies: "@types/react" "*" @@ -1083,26 +1076,21 @@ acorn-globals@^7.0.0: acorn "^8.1.0" acorn-walk "^8.0.2" -acorn-import-assertions@^1.7.6: - version "1.8.0" - resolved "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz" - integrity sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw== +acorn-import-assertions@^1.9.0: + version "1.9.0" + resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz#507276249d684797c84e0734ef84860334cfb1ac" + integrity sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA== acorn-walk@^8.0.2: version "8.2.0" resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz" integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== -acorn@^8.1.0, acorn@^8.8.1: +acorn@^8.1.0, acorn@^8.5.0, acorn@^8.7.1, acorn@^8.8.1: version "8.8.2" resolved "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== -acorn@^8.5.0, acorn@^8.7.1: - version "8.8.0" - resolved "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz" - integrity sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w== - agent-base@6: version "6.0.2" resolved "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz" @@ -1273,18 +1261,7 @@ braces@^3.0.1, braces@^3.0.2: dependencies: fill-range "^7.0.1" -browserslist@^4.14.5: - version "4.17.5" - resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.17.5.tgz" - integrity sha512-I3ekeB92mmpctWBoLXe0d5wPS2cBuRvvW0JyyJHMrk9/HmP2ZjrTboNAZ8iuGqaEIlKguljbQY32OkOJIRrgoA== - dependencies: - caniuse-lite "^1.0.30001271" - electron-to-chromium "^1.3.878" - escalade "^3.1.1" - node-releases "^2.0.1" - picocolors "^1.0.0" - -browserslist@^4.21.3: +browserslist@^4.14.5, browserslist@^4.21.3: version "4.21.5" resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz" integrity sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w== @@ -1336,11 +1313,6 @@ camelcase@^6.2.0: resolved "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -caniuse-lite@^1.0.30001271: - version "1.0.30001274" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001274.tgz" - integrity sha512-+Nkvv0fHyhISkiMIjnyjmf5YJcQ1IQHZN6U9TLUMroWR38FNwpsC51Gb68yueafX1V6ifOisInSgP9WJFS13ew== - caniuse-lite@^1.0.30001449: version "1.0.30001465" resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001465.tgz" @@ -1615,11 +1587,6 @@ domexception@^4.0.0: dependencies: webidl-conversions "^7.0.0" -electron-to-chromium@^1.3.878: - version "1.3.886" - resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.886.tgz" - integrity sha512-+vYdeBosI63VkCtNWnEVFjgNd/IZwvnsWkKyPtWAvrhA+XfByKoBJcbsMgudVU/bUcGAF9Xp3aXn96voWlc3oQ== - electron-to-chromium@^1.4.284: version "1.4.328" resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.328.tgz" @@ -1635,10 +1602,10 @@ emoji-regex@^8.0.0: resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== -enhanced-resolve@^5.0.0, enhanced-resolve@^5.13.0: - version "5.13.0" - resolved "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.13.0.tgz" - integrity sha512-eyV8f0y1+bzyfh8xAwW/WTSZpLbjhqc4ne9eGSH4Zo2ejdyiNG9pU6mf9DG8a7+Auk6MFTlNOT4Y2y/9k8GKVg== +enhanced-resolve@^5.0.0, enhanced-resolve@^5.15.0: + version "5.15.0" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz#1af946c7d93603eb88e9896cee4904dc012e9c35" + integrity sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg== dependencies: graceful-fs "^4.2.4" tapable "^2.2.0" @@ -1914,12 +1881,7 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" -graceful-fs@^4.1.2: - version "4.2.8" - resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz" - integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== - -graceful-fs@^4.2.4, graceful-fs@^4.2.9: +graceful-fs@^4.1.2, graceful-fs@^4.2.4, graceful-fs@^4.2.9: version "4.2.10" resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== @@ -2847,30 +2809,18 @@ micromatch@^4.0.4: braces "^3.0.2" picomatch "^2.3.1" -mime-db@1.50.0: - version "1.50.0" - resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.50.0.tgz" - integrity sha512-9tMZCDlYHqeERXEHO9f/hKfNXhre5dK2eE/krIvUjZbS2KPcqGDfNShIWS1uW9XOTKQKqK6qbeOci18rbfW77A== - mime-db@1.52.0: version "1.52.0" resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.12: +mime-types@^2.1.12, mime-types@^2.1.27: version "2.1.35" resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== dependencies: mime-db "1.52.0" -mime-types@^2.1.27: - version "2.1.33" - resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.33.tgz" - integrity sha512-plLElXp7pRDd0bNZHw+nMd52vRYjLwQjygaNg7ddJ2uJtTlmnTCjWuPKxVu6//AdaRuME84SvLW91sIkBqGT0g== - dependencies: - mime-db "1.50.0" - mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" @@ -2903,11 +2853,6 @@ node-int64@^0.4.0: resolved "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz" integrity sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw== -node-releases@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.1.tgz" - integrity sha512-CqyzN6z7Q6aMeF/ktcMVTzhAHCEpf8SOarwpzpf8pNBY2k5/oM34UHldUwp8VKI7uxct2HxSRdJjBaZeESzcxA== - node-releases@^2.0.8: version "2.0.10" resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz" @@ -3313,10 +3258,10 @@ scheduler@^0.23.0: dependencies: loose-envify "^1.1.0" -schema-utils@^3.1.1, schema-utils@^3.1.2: - version "3.1.2" - resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.2.tgz" - integrity sha512-pvjEHOgWc9OWA/f/DE3ohBWTD6EleVLf7iFUkoSwAxttdBhB9QUebQgxER2kWueOvRJXPHNnyrvvh9eZINB8Eg== +schema-utils@^3.1.1, schema-utils@^3.2.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.3.0.tgz#f50a88877c3c01652a15b622ae9e9795df7a60fe" + integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== dependencies: "@types/json-schema" "^7.0.8" ajv "^6.12.5" @@ -3738,10 +3683,10 @@ webpack-sources@^3.2.3: resolved "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz" integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== -webpack@^5.74.0: - version "5.81.0" - resolved "https://registry.npmjs.org/webpack/-/webpack-5.81.0.tgz" - integrity sha512-AAjaJ9S4hYCVODKLQTgG5p5e11hiMawBwV2v8MYLE0C/6UAGLuAF4n1qa9GOwdxnicaP+5k6M5HrLmD4+gIB8Q== +webpack@^5.88.0: + version "5.88.0" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.88.0.tgz#a07aa2f8e7a64a8f1cec0c6c2e180e3cb34440c8" + integrity sha512-O3jDhG5e44qIBSi/P6KpcCcH7HD+nYIHVBhdWFxcLOcIGN8zGo5nqF3BjyNCxIh4p1vFdNnreZv2h2KkoAw3lw== dependencies: "@types/eslint-scope" "^3.7.3" "@types/estree" "^1.0.0" @@ -3749,10 +3694,10 @@ webpack@^5.74.0: "@webassemblyjs/wasm-edit" "^1.11.5" "@webassemblyjs/wasm-parser" "^1.11.5" acorn "^8.7.1" - acorn-import-assertions "^1.7.6" + acorn-import-assertions "^1.9.0" browserslist "^4.14.5" chrome-trace-event "^1.0.2" - enhanced-resolve "^5.13.0" + enhanced-resolve "^5.15.0" es-module-lexer "^1.2.1" eslint-scope "5.1.1" events "^3.2.0" @@ -3762,7 +3707,7 @@ webpack@^5.74.0: loader-runner "^4.2.0" mime-types "^2.1.27" neo-async "^2.6.2" - schema-utils "^3.1.2" + schema-utils "^3.2.0" tapable "^2.1.1" terser-webpack-plugin "^5.3.7" watchpack "^2.4.0" diff --git a/snuba/cli/consumer.py b/snuba/cli/consumer.py index d77bba7da8..d6e242a268 100644 --- a/snuba/cli/consumer.py +++ b/snuba/cli/consumer.py @@ -116,6 +116,13 @@ type=int, ) @click.option("--join-timeout", type=int, help="Join timeout in seconds.", default=5) +@click.option( + "--enforce-schema", + type=bool, + is_flag=True, + default=False, + help="Enforce schema on the raw events topic.", +) @click.option( "--profile-path", type=click.Path(dir_okay=True, file_okay=False, exists=True) ) @@ -144,6 +151,7 @@ def consumer( input_block_size: Optional[int], output_block_size: Optional[int], join_timeout: int = 5, + enforce_schema: bool = False, log_level: Optional[str] = None, profile_path: Optional[str] = None, max_poll_interval_ms: Optional[int] = None, @@ -201,6 +209,7 @@ def consumer( slice_id=slice_id, join_timeout=join_timeout, max_poll_interval_ms=max_poll_interval_ms, + enforce_schema=enforce_schema, ) consumer = consumer_builder.build_base_consumer() diff --git a/snuba/cli/devserver.py b/snuba/cli/devserver.py index 15c7280600..b66a824710 100644 --- a/snuba/cli/devserver.py +++ b/snuba/cli/devserver.py @@ -46,6 +46,7 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: "--log-level=debug", "--storage=transactions", "--consumer-group=transactions_group", + "--enforce-schema", ], ), ( @@ -70,6 +71,7 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: "--log-level=debug", "--storage=outcomes_raw", "--consumer-group=outcomes_group", + "--enforce-schema", ], ), ( @@ -81,6 +83,7 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: "--no-strict-offset-reset", "--log-level=debug", "--storage=errors", + "--enforce-schema", ], ), ( @@ -213,6 +216,7 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: "--no-strict-offset-reset", "--log-level=debug", "--consumer-group=snuba-metrics-consumers", + "--enforce-schema", ], ), ( @@ -225,6 +229,7 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: "--no-strict-offset-reset", "--log-level=debug", "--consumer-group=snuba-gen-metrics-distributions-consumers", + "--enforce-schema", ], ), ( @@ -237,6 +242,7 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: "--no-strict-offset-reset", "--log-level=debug", "--consumer-group=snuba-gen-metrics-sets-consumers", + "--enforce-schema", ], ), ( @@ -249,6 +255,7 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: "--no-strict-offset-reset", "--log-level=debug", "--consumer-group=snuba-gen-metrics-counters-consumers", + "--enforce-schema", ], ), ] @@ -428,6 +435,7 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: "--log-level=debug", "--storage=spans", "--consumer-group=spans_group", + "--enforce-schema", ], ), ] diff --git a/snuba/cli/dlq_consumer.py b/snuba/cli/dlq_consumer.py index 10afcd2786..c6c12c065c 100644 --- a/snuba/cli/dlq_consumer.py +++ b/snuba/cli/dlq_consumer.py @@ -169,6 +169,7 @@ def handler(signum: int, frame: Any) -> None: metrics=metrics, slice_id=instruction.slice_id, join_timeout=None, + enforce_schema=False, ) consumer = consumer_builder.build_dlq_consumer(instruction) diff --git a/snuba/consumers/consumer.py b/snuba/consumers/consumer.py index e8137bfb7b..f8215721e2 100644 --- a/snuba/consumers/consumer.py +++ b/snuba/consumers/consumer.py @@ -523,6 +523,7 @@ def process_message( processor: MessageProcessor, consumer_group: str, snuba_logical_topic: SnubaTopic, + enforce_schema: bool, message: Message[KafkaPayload], ) -> Union[None, BytesInsertBatch, ReplacementBatch]: local_metrics = MetricsWrapper( @@ -573,6 +574,8 @@ def process_message( _LAST_INVALID_MESSAGE[snuba_logical_topic.name] = start sentry_sdk.set_tag("invalid_message_schema", "true") logger.warning(err, exc_info=True) + if enforce_schema: + raise # TODO: this is not the most efficient place to emit a metric, but # as long as should_validate is behind a sample rate it should be @@ -604,8 +607,6 @@ def process_message( value = message.value raise InvalidMessage(value.partition, value.offset) from err - return None - if isinstance(result, InsertBatch): return BytesInsertBatch( [json_row_encoder.encode(row) for row in result.rows], diff --git a/snuba/consumers/consumer_builder.py b/snuba/consumers/consumer_builder.py index be6285de46..77f98fb6b6 100644 --- a/snuba/consumers/consumer_builder.py +++ b/snuba/consumers/consumer_builder.py @@ -70,7 +70,8 @@ def __init__( max_batch_time_ms: int, metrics: MetricsBackend, slice_id: Optional[int], - join_timeout: Optional[int], + join_timeout: Optional[float], + enforce_schema: bool, profile_path: Optional[str] = None, max_poll_interval_ms: Optional[int] = None, ) -> None: @@ -83,6 +84,7 @@ def __init__( self.__consumer_config = consumer_config self.__kafka_params = kafka_params self.consumer_group = kafka_params.group_id + self.__enforce_schema = enforce_schema broker_config = build_kafka_consumer_configuration( self.__consumer_config.raw_topic.broker_config, @@ -146,6 +148,7 @@ def __init__( def __build_consumer( self, strategy_factory: ProcessingStrategyFactory[KafkaPayload], + input_topic: Topic, dlq_policy: Optional[DlqPolicy[KafkaPayload]], ) -> StreamProcessor[KafkaPayload]: @@ -179,7 +182,7 @@ def log_general_error(e: KafkaError) -> None: return StreamProcessor( consumer, - self.raw_topic, + input_topic, strategy_factory, IMMEDIATE, dlq_policy=dlq_policy, @@ -212,6 +215,7 @@ def build_streaming_strategy_factory( processor, self.consumer_group, logical_topic, + self.__enforce_schema, ), collector=build_batch_writer( table_writer, @@ -299,7 +303,9 @@ def build_base_consumer(self) -> StreamProcessor[KafkaPayload]: Builds the consumer. """ return self.__build_consumer( - self.build_streaming_strategy_factory(), self.__build_default_dlq_policy() + self.build_streaming_strategy_factory(), + self.raw_topic, + self.__build_default_dlq_policy(), ) def build_dlq_consumer( @@ -325,8 +331,13 @@ def build_dlq_consumer( else: raise ValueError("Invalid DLQ policy") + dlq_topic = self.__consumer_config.dlq_topic + assert dlq_topic is not None + return self.__build_consumer( - self.build_dlq_strategy_factory(instruction), dlq_policy + self.build_dlq_strategy_factory(instruction), + Topic(dlq_topic.physical_topic_name), + dlq_policy, ) def __build_default_dlq_policy(self) -> Optional[DlqPolicy[KafkaPayload]]: diff --git a/snuba/consumers/dlq.py b/snuba/consumers/dlq.py index 476130d39e..8b1409050b 100644 --- a/snuba/consumers/dlq.py +++ b/snuba/consumers/dlq.py @@ -8,6 +8,7 @@ from typing import Optional, TypeVar import rapidjson +from arroyo.dlq import InvalidMessage from arroyo.processing.strategies.abstract import ProcessingStrategy from arroyo.types import Message @@ -150,7 +151,12 @@ def poll(self) -> None: def submit(self, message: Message[TPayload]) -> None: if self.__processed_messages < self.__num_messages_to_process: self.__last_message_time = time.time() - self.__next_step.submit(message) + + try: + self.__next_step.submit(message) + except InvalidMessage: + self.__processed_messages += 1 + raise self.__processed_messages += 1 def close(self) -> None: diff --git a/snuba/datasets/processors/outcomes_processor.py b/snuba/datasets/processors/outcomes_processor.py index 4fe530b7ea..4485b1d42b 100644 --- a/snuba/datasets/processors/outcomes_processor.py +++ b/snuba/datasets/processors/outcomes_processor.py @@ -33,6 +33,8 @@ "sample_rate", "send_error", "internal_sdk_error", + "insufficient_data", + "backpressure", ] ) diff --git a/snuba/datasets/processors/querylog_processor.py b/snuba/datasets/processors/querylog_processor.py index dacb93ae0c..92160b1153 100644 --- a/snuba/datasets/processors/querylog_processor.py +++ b/snuba/datasets/processors/querylog_processor.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import random import uuid from typing import Any, Mapping, MutableMapping, Optional, Sequence, Union @@ -11,7 +12,7 @@ QueryMetadata, ) -from snuba import environment +from snuba import environment, state from snuba.consumers.types import KafkaMessageMetadata from snuba.datasets.processors import DatasetMessageProcessor from snuba.processor import InsertBatch, ProcessedMessage @@ -150,6 +151,12 @@ def _remove_invalid_data(self, processed: dict[str, Any]) -> None: def process_message( self, message: Querylog, metadata: KafkaMessageMetadata ) -> Optional[ProcessedMessage]: + # XXX: Temporary code for the DLQ test. + reject_rate = state.get_config("querylog_reject_rate", 0.0) + assert isinstance(reject_rate, float) + if random.random() < reject_rate: + raise ValueError("This message is rejected on purpose.") + processed = { "request_id": str(uuid.UUID(message["request"]["id"])), "request_body": self.__to_json_string(message["request"]["body"]), diff --git a/snuba/datasets/processors/search_issues_processor.py b/snuba/datasets/processors/search_issues_processor.py index c0c65b0f8d..ee5ec5cb64 100644 --- a/snuba/datasets/processors/search_issues_processor.py +++ b/snuba/datasets/processors/search_issues_processor.py @@ -327,7 +327,7 @@ def process_message( processed = self.process_insert_v1(event, metadata) except EventTooOld: metrics.increment("event_too_old") - raise + return None except IndexError: metrics.increment("invalid_message") raise diff --git a/snuba/redis.py b/snuba/redis.py index d8efe15043..575948018e 100644 --- a/snuba/redis.py +++ b/snuba/redis.py @@ -113,6 +113,7 @@ class RedisClientKey(Enum): CONFIG = "config" DLQ = "dlq" OPTIMIZE = "optimize" + ADMIN_AUTH = "admin_auth" _redis_clients: Mapping[RedisClientKey, RedisClientType] = { @@ -137,6 +138,9 @@ class RedisClientKey(Enum): RedisClientKey.OPTIMIZE: _initialize_specialized_redis_cluster( settings.REDIS_CLUSTERS["optimize"] ), + RedisClientKey.ADMIN_AUTH: _initialize_specialized_redis_cluster( + settings.REDIS_CLUSTERS["admin_auth"] + ), } diff --git a/snuba/settings/__init__.py b/snuba/settings/__init__.py index 7533ac0550..1a62cd1205 100644 --- a/snuba/settings/__init__.py +++ b/snuba/settings/__init__.py @@ -55,7 +55,9 @@ os.environ.get("ADMIN_REPLAYS_SAMPLE_RATE_ON_ERROR", 1.0) ) + ADMIN_ALLOWED_PROD_PROJECTS: Sequence[int] = [] +ADMIN_ROLES_REDIS_TTL = 600 ###################### # End Admin Settings # @@ -156,6 +158,7 @@ class RedisClusters(TypedDict): config: RedisClusterConfig | None dlq: RedisClusterConfig | None optimize: RedisClusterConfig | None + admin_auth: RedisClusterConfig | None REDIS_CLUSTERS: RedisClusters = { @@ -166,6 +169,7 @@ class RedisClusters(TypedDict): "config": None, "dlq": None, "optimize": None, + "admin_auth": None, } # Query Recording Options diff --git a/snuba/settings/settings_test.py b/snuba/settings/settings_test.py index e343402823..44af17603f 100644 --- a/snuba/settings/settings_test.py +++ b/snuba/settings/settings_test.py @@ -57,5 +57,6 @@ (6, "config"), (7, "dlq"), (8, "optimize"), + (9, "admin_auth"), ] } diff --git a/snuba/web/views.py b/snuba/web/views.py index aac3a1d23a..7cbde9d50e 100644 --- a/snuba/web/views.py +++ b/snuba/web/views.py @@ -621,6 +621,9 @@ def eventstream(*, entity: Entity) -> RespTuple: storage = entity.get_writable_storage() assert storage is not None + storage_name = storage.get_storage_key().value + # TODO: A few storages are currently excluded from schema validation + should_validate_schema = storage_name not in ["search_issues", "replays"] try: if type_ == "insert": @@ -644,6 +647,7 @@ def commit( stream_loader.get_processor(), "consumer_grouup", stream_loader.get_default_topic_spec().topic, + should_validate_schema, ), build_batch_writer(table_writer, metrics=metrics), max_batch_size=1, diff --git a/tests/admin/clickhouse_migrations/test_api.py b/tests/admin/clickhouse_migrations/test_api.py index 9140e8385e..b7d688fd42 100644 --- a/tests/admin/clickhouse_migrations/test_api.py +++ b/tests/admin/clickhouse_migrations/test_api.py @@ -28,6 +28,7 @@ from snuba.migrations.policies import MigrationPolicy from snuba.migrations.runner import MigrationKey, Runner from snuba.migrations.status import Status +from snuba.redis import RedisClientKey, get_redis_client def generate_migration_test_role( @@ -60,6 +61,7 @@ def admin_api() -> FlaskClient: return application.test_client() +@pytest.mark.redis_db @pytest.mark.clickhouse_db def test_migration_groups(admin_api: FlaskClient) -> None: runner = Runner() @@ -105,6 +107,7 @@ def get_migration_ids( ] +@pytest.mark.redis_db @pytest.mark.clickhouse_db def test_list_migration_status(admin_api: FlaskClient) -> None: with patch( @@ -166,6 +169,7 @@ def sort_by_migration_id(migration: Any) -> Any: assert sorted_response == sorted_expected_json +@pytest.mark.redis_db @pytest.mark.clickhouse_db @pytest.mark.parametrize("action", ["run", "reverse"]) def test_run_reverse_migrations(admin_api: FlaskClient, action: str) -> None: @@ -310,6 +314,7 @@ def print_something(*args: Any, **kwargs: Any) -> None: assert mock_run_migration.call_count == 1 +@pytest.mark.redis_db def test_get_iam_roles(caplog: Any) -> None: system_role = generate_migration_test_role("system", "all") tool_role = generate_tool_test_role("snql-to-sql") @@ -388,6 +393,8 @@ def test_get_iam_roles(caplog: Any) -> None: tool_role, ] + iam_file.close() + with patch( "snuba.admin.auth.settings.ADMIN_IAM_POLICY_FILE", "file_not_exists.json" ): @@ -398,3 +405,122 @@ def test_get_iam_roles(caplog: Any) -> None: assert "IAM policy file not found file_not_exists.json" in str( log.calls ) + + +@pytest.mark.redis_db +def test_get_iam_roles_cache() -> None: + system_role = generate_migration_test_role("system", "all") + tool_role = generate_tool_test_role("snql-to-sql") + with patch( + "snuba.admin.auth.DEFAULT_ROLES", + [system_role, tool_role], + ): + iam_file = tempfile.NamedTemporaryFile() + iam_file.write( + json.dumps( + { + "bindings": [ + { + "members": [ + "group:team-sns@sentry.io", + "user:test_user1@sentry.io", + ], + "role": "roles/NonBlockingMigrationsExecutor", + }, + { + "members": [ + "group:team-sns@sentry.io", + "user:test_user1@sentry.io", + "user:test_user2@sentry.io", + ], + "role": "roles/TestMigrationsExecutor", + }, + { + "members": [ + "group:team-sns@sentry.io", + "user:test_user1@sentry.io", + "user:test_user2@sentry.io", + ], + "role": "roles/owner", + }, + { + "members": [ + "group:team-sns@sentry.io", + "user:test_user1@sentry.io", + ], + "role": "roles/AllTools", + }, + ] + } + ).encode("utf-8") + ) + + iam_file.flush() + with patch("snuba.admin.auth.settings.ADMIN_IAM_POLICY_FILE", iam_file.name): + + user1 = AdminUser(email="test_user1@sentry.io", id="unknown") + _set_roles(user1) + + assert user1.roles == [ + ROLES["NonBlockingMigrationsExecutor"], + ROLES["TestMigrationsExecutor"], + ROLES["AllTools"], + system_role, + tool_role, + ] + + iam_file = tempfile.NamedTemporaryFile() + iam_file.write(json.dumps({"bindings": []}).encode("utf-8")) + iam_file.flush() + + with patch("snuba.admin.auth.settings.ADMIN_IAM_POLICY_FILE", iam_file.name): + _set_roles(user1) + + assert user1.roles == [ + ROLES["NonBlockingMigrationsExecutor"], + ROLES["TestMigrationsExecutor"], + ROLES["AllTools"], + system_role, + tool_role, + ] + + redis_client = get_redis_client(RedisClientKey.ADMIN_AUTH) + redis_client.delete(f"roles-{user1.email}") + _set_roles(user1) + + assert user1.roles == [ + system_role, + tool_role, + ] + + +@pytest.mark.redis_db +@patch("redis.Redis") +def test_get_iam_roles_cache_fail(mock_redis: Any) -> None: + mock_redis.get.side_effect = Exception("Test exception") + mock_redis.set.side_effect = Exception("Test exception") + system_role = generate_migration_test_role("system", "all") + tool_role = generate_tool_test_role("snql-to-sql") + with patch( + "snuba.admin.auth.DEFAULT_ROLES", + [system_role, tool_role], + ): + iam_file = tempfile.NamedTemporaryFile() + iam_file.write(json.dumps({"bindings": []}).encode("utf-8")) + iam_file.flush() + + with patch("snuba.admin.auth.settings.ADMIN_IAM_POLICY_FILE", iam_file.name): + user1 = AdminUser(email="test_user1@sentry.io", id="unknown") + _set_roles(user1) + + assert user1.roles == [ + system_role, + tool_role, + ] + + _set_roles(user1) + + assert user1.roles == [ + system_role, + tool_role, + ] diff --git a/tests/admin/test_api.py b/tests/admin/test_api.py index a2df448f23..e62e3e86cd 100644 --- a/tests/admin/test_api.py +++ b/tests/admin/test_api.py @@ -166,6 +166,7 @@ def test_config_descriptions(admin_api: FlaskClient) -> None: } +@pytest.mark.redis_db def get_node_for_table( admin_api: FlaskClient, storage_name: str ) -> tuple[str, str, int]: @@ -204,6 +205,7 @@ def test_system_query(admin_api: FlaskClient) -> None: assert data["rows"] == [] +@pytest.mark.redis_db def test_predefined_system_queries(admin_api: FlaskClient) -> None: response = admin_api.get( "/clickhouse_queries", @@ -249,6 +251,7 @@ def test_query_trace_bad_query(admin_api: FlaskClient) -> None: assert "clickhouse" == data["error"]["type"] +@pytest.mark.redis_db @pytest.mark.clickhouse_db def test_query_trace_invalid_query(admin_api: FlaskClient) -> None: table, _, _ = get_node_for_table(admin_api, "errors_ro") @@ -279,6 +282,7 @@ def test_querylog_query(admin_api: FlaskClient) -> None: assert "column_names" in data and data["column_names"] == ["count()"] +@pytest.mark.redis_db @pytest.mark.clickhouse_db def test_querylog_invalid_query(admin_api: FlaskClient) -> None: table, _, _ = get_node_for_table(admin_api, "errors_ro") @@ -301,6 +305,7 @@ def test_querylog_describe(admin_api: FlaskClient) -> None: assert "column_names" in data and "rows" in data +@pytest.mark.redis_db def test_predefined_querylog_queries(admin_api: FlaskClient) -> None: response = admin_api.get( "/querylog_queries", @@ -313,6 +318,7 @@ def test_predefined_querylog_queries(admin_api: FlaskClient) -> None: assert data[0]["name"] == "QueryByID" +@pytest.mark.redis_db def test_get_snuba_datasets(admin_api: FlaskClient) -> None: response = admin_api.get("/snuba_datasets") assert response.status_code == 200 @@ -320,6 +326,7 @@ def test_get_snuba_datasets(admin_api: FlaskClient) -> None: assert set(data) == set(get_enabled_dataset_names()) +@pytest.mark.redis_db def test_convert_SnQL_to_SQL_invalid_dataset(admin_api: FlaskClient) -> None: response = admin_api.post( "/snql_to_sql", data=json.dumps({"dataset": "", "query": ""}) @@ -329,6 +336,7 @@ def test_convert_SnQL_to_SQL_invalid_dataset(admin_api: FlaskClient) -> None: assert data["error"]["message"] == "dataset '' does not exist" +@pytest.mark.redis_db @pytest.mark.redis_db def test_convert_SnQL_to_SQL_invalid_query(admin_api: FlaskClient) -> None: response = admin_api.post( diff --git a/tests/admin/test_authorization.py b/tests/admin/test_authorization.py index 923902a61e..d07614435f 100644 --- a/tests/admin/test_authorization.py +++ b/tests/admin/test_authorization.py @@ -16,6 +16,7 @@ def admin_api() -> FlaskClient: return application.test_client() +@pytest.mark.redis_db def test_tools(admin_api: FlaskClient) -> None: response = admin_api.get("/tools") assert response.status_code == 200 @@ -25,6 +26,7 @@ def test_tools(admin_api: FlaskClient) -> None: assert "all" in data["tools"] +@pytest.mark.redis_db @patch("snuba.admin.auth.DEFAULT_ROLES", [ROLES["ProductTools"]]) def test_product_tools_role( admin_api: FlaskClient, diff --git a/tests/consumers/test_consumer_builder.py b/tests/consumers/test_consumer_builder.py index c76084c006..c8d9c1d0cb 100644 --- a/tests/consumers/test_consumer_builder.py +++ b/tests/consumers/test_consumer_builder.py @@ -18,7 +18,7 @@ from snuba.datasets.storages.storage_key import StorageKey from snuba.utils.metrics.backends.abstract import MetricsBackend from snuba.utils.metrics.wrapper import MetricsWrapper -from tests.fixtures import get_raw_event +from tests.fixtures import get_raw_error_message from tests.test_consumer import get_row_count test_storage_key = StorageKey("errors") @@ -61,6 +61,7 @@ ), slice_id=None, join_timeout=5, + enforce_schema=True, ) optional_consumer_config = resolve_consumer_config( @@ -104,6 +105,7 @@ ), slice_id=None, join_timeout=5, + enforce_schema=True, ) @@ -160,8 +162,7 @@ def test_run_processing_strategy() -> None: strategy_factory = consumer_builder.build_streaming_strategy_factory() strategy = strategy_factory.create_with_partitions(commit, partitions) - raw_message = get_raw_event() - json_string = json.dumps([2, "insert", raw_message, []]) + json_string = json.dumps(get_raw_error_message()) message = Message( BrokerValue( diff --git a/tests/test_consumer.py b/tests/test_consumer.py index 582c8eb5fd..dd7fa9e99f 100644 --- a/tests/test_consumer.py +++ b/tests/test_consumer.py @@ -31,13 +31,16 @@ from snuba.utils.streams.topics import Topic as SnubaTopic from tests.assertions import assert_changes from tests.backends.metrics import TestingMetricsBackend, Timing +from tests.fixtures import get_raw_error_message def test_streaming_consumer_strategy() -> None: messages = ( Message( BrokerValue( - KafkaPayload(None, b"{}", []), + KafkaPayload( + None, json.dumps(get_raw_error_message()).encode("utf-8"), [] + ), Partition(Topic("events"), 0), i, datetime.now(), @@ -72,7 +75,7 @@ def write_step() -> ProcessedMessageBatchWriter: factory = KafkaConsumerStrategyFactory( None, functools.partial( - process_message, processor, "consumer_group", SnubaTopic.EVENTS + process_message, processor, "consumer_group", SnubaTopic.EVENTS, True ), write_step, max_batch_size=10,