From 04360877f6e9e36c6f37ad83209808364fd4bda3 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Thu, 18 Jul 2024 11:50:22 -0400 Subject: [PATCH 01/17] Add processor for new spans schema --- rust_snuba/Cargo.lock | 418 +++++++++++++++--- rust_snuba/Cargo.toml | 1 + rust_snuba/src/processors/mod.rs | 2 + rust_snuba/src/processors/spans.rs | 48 +- rust_snuba/src/processors/spans_v2.rs | 168 +++++++ .../datasets/processors/spans_v2_processor.py | 6 + 6 files changed, 548 insertions(+), 95 deletions(-) create mode 100644 rust_snuba/src/processors/spans_v2.rs create mode 100644 snuba/datasets/processors/spans_v2_processor.py diff --git a/rust_snuba/Cargo.lock b/rust_snuba/Cargo.lock index b49c6bfe8e..3c1cec3adf 100644 --- a/rust_snuba/Cargo.lock +++ b/rust_snuba/Cargo.lock @@ -27,7 +27,7 @@ dependencies = [ "getrandom 0.2.14", "once_cell", "serde", - "version_check", + "version_check 0.9.4", "zerocopy", ] @@ -67,6 +67,15 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "anstream" version = "0.6.13" @@ -149,7 +158,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" dependencies = [ - "quote", + "quote 1.0.36", "syn 1.0.109", ] @@ -349,8 +358,8 @@ version = "0.1.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] @@ -360,6 +369,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi 0.3.9", +] + [[package]] name = "autocfg" version = "1.2.0" @@ -414,6 +434,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.49.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c07087f3d5731bf3fb375a81841b99597e25dc11bd3bc72d16d43adf6624a6e" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "cfg-if 0.1.10", + "clang-sys", + "clap 2.34.0", + "env_logger 0.6.2", + "fxhash", + "lazy_static", + "log", + "peeking_take_while", + "proc-macro2 0.4.30", + "quote 0.6.13", + "regex", + "shlex", + "which", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -478,6 +521,12 @@ version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.6.0" @@ -509,6 +558,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cexpr" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d" +dependencies = [ + "nom 4.2.3", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -569,6 +627,46 @@ dependencies = [ "half", ] +[[package]] +name = "cityhash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ecb82e5b6394ea8408bc8f454d50c75f339ca89f9af90a6ff8694e6a1c63da" +dependencies = [ + "bindgen", + "cc", + "clap 2.34.0", + "libc", +] + +[[package]] +name = "clang-sys" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "term_size", + "textwrap", + "unicode-width", + "vec_map", + "yaml-rust", +] + [[package]] name = "clap" version = "4.5.4" @@ -588,7 +686,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] @@ -598,8 +696,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ "heck 0.5.0", - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] @@ -699,7 +797,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap", + "clap 4.5.4", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -782,7 +880,7 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ - "quote", + "quote 1.0.36", "syn 1.0.109", ] @@ -814,9 +912,9 @@ checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" dependencies = [ "fnv", "ident_case", - "proc-macro2", - "quote", - "strsim", + "proc-macro2 1.0.79", + "quote 1.0.36", + "strsim 0.11.1", "syn 2.0.58", ] @@ -827,7 +925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ "darling_core", - "quote", + "quote 1.0.36", "syn 2.0.58", ] @@ -853,8 +951,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a3767f826efbbe5a5ae093920b58b43b01734202be697e1354914e862e8e704" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "semver", "syn 2.0.58", ] @@ -936,13 +1034,26 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "env_logger" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" +dependencies = [ + "atty", + "humantime 1.3.0", + "log", + "regex", + "termcolor", +] + [[package]] name = "env_logger" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ - "humantime", + "humantime 2.1.0", "is-terminal", "log", "regex", @@ -1024,6 +1135,15 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", +] + [[package]] name = "fancy-regex" version = "0.11.0" @@ -1205,8 +1325,8 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] @@ -1240,6 +1360,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1247,7 +1376,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", - "version_check", + "version_check 0.9.4", ] [[package]] @@ -1280,6 +1409,12 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "gloo-timers" version = "0.2.6" @@ -1358,6 +1493,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.3.9" @@ -1464,6 +1608,15 @@ dependencies = [ "url", ] +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + [[package]] name = "humantime" version = "2.1.0" @@ -1625,7 +1778,7 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "windows-sys 0.48.0", ] @@ -1670,7 +1823,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "windows-sys 0.52.0", ] @@ -1681,7 +1834,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -1749,7 +1902,7 @@ dependencies = [ "anyhow", "base64 0.21.7", "bytecount", - "clap", + "clap 4.5.4", "fancy-regex", "fraction", "getrandom 0.2.14", @@ -1806,7 +1959,7 @@ dependencies = [ "string_cache", "term", "tiny-keccak", - "unicode-xid", + "unicode-xid 0.2.4", "walkdir", ] @@ -1837,6 +1990,16 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libloading" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" +dependencies = [ + "cc", + "winapi 0.3.9", +] + [[package]] name = "libredox" version = "0.1.3" @@ -2042,6 +2205,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +dependencies = [ + "memchr", + "version_check 0.1.5", +] + [[package]] name = "nom" version = "7.1.3" @@ -2155,7 +2328,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -2175,8 +2348,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ "proc-macro-crate", - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 1.0.109", ] @@ -2222,8 +2395,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] @@ -2306,6 +2479,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2341,8 +2520,8 @@ checksum = "c35eeed0a3fab112f75165fdc026b3913f4183133f19b49be773ac9ea966e8bd" dependencies = [ "pest", "pest_meta", - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] @@ -2463,7 +2642,7 @@ checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" dependencies = [ "cfg-if 1.0.0", "concurrent-queue", - "hermit-abi", + "hermit-abi 0.3.9", "pin-project-lite", "rustix 0.38.32", "tracing", @@ -2494,7 +2673,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.79", "syn 2.0.58", ] @@ -2508,6 +2687,15 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.79" @@ -2577,9 +2765,9 @@ version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d39c55dab3fc5a4b25bbd1ac10a2da452c4aca13bb450f22818a002e29648d" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.79", "pyo3-macros-backend", - "quote", + "quote 1.0.36", "syn 1.0.109", ] @@ -2589,18 +2777,33 @@ version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97daff08a4c48320587b5224cc98d609e3c27b6d437315bd40b605c98eeb5918" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 1.0.109", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.79", ] [[package]] @@ -2878,6 +3081,7 @@ dependencies = [ "anyhow", "cadence", "chrono", + "cityhash", "criterion", "ctrlc", "data-encoding", @@ -3018,8 +3222,8 @@ version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "serde_derive_internals", "syn 1.0.109", ] @@ -3226,8 +3430,8 @@ version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] @@ -3237,8 +3441,8 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 1.0.109", ] @@ -3279,8 +3483,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a00ffd23fd882d096f09fcaae2a9de8329a328628e86027e049ee051dc1621f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "serde", "syn 2.0.58", ] @@ -3322,8 +3526,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" dependencies = [ "darling", - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] @@ -3360,6 +3564,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" + [[package]] name = "signal-hook" version = "0.3.17" @@ -3434,9 +3644,9 @@ checksum = "793ec303cef342e5a80e717016050673430851d5e3159ba8275c0f801cc83c11" dependencies = [ "anyhow", "cadence", - "clap", + "clap 4.5.4", "crc32fast", - "env_logger", + "env_logger 0.10.2", "log", "rand 0.8.5", "sentry", @@ -3459,6 +3669,12 @@ dependencies = [ "precomputed-hash", ] +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + [[package]] name = "strsim" version = "0.11.1" @@ -3471,8 +3687,8 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "unicode-ident", ] @@ -3482,8 +3698,8 @@ version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "unicode-ident", ] @@ -3543,6 +3759,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "term_size" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -3552,6 +3778,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "term_size", + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.58" @@ -3567,8 +3803,8 @@ version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] @@ -3692,8 +3928,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] @@ -3772,8 +4008,8 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] @@ -3859,8 +4095,8 @@ checksum = "95e32f38493804f88e2dc7a5412eccd872ea5452b4db9b0a77de4df180f2a87e" dependencies = [ "heck 0.4.1", "log", - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "regress 0.8.0", "schemars", "serde_json", @@ -3875,8 +4111,8 @@ version = "0.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc09508b72f63d521d68e42c7f172c7416d67986df44b3c7d1f7f9963948ed32" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "schemars", "serde", "serde_json", @@ -3921,6 +4157,18 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.4" @@ -3998,6 +4246,18 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" + [[package]] name = "version_check" version = "0.9.4" @@ -4069,8 +4329,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", "wasm-bindgen-shared", ] @@ -4093,7 +4353,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ - "quote", + "quote 1.0.36", "wasm-bindgen-macro-support", ] @@ -4103,8 +4363,8 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -4139,6 +4399,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164" +dependencies = [ + "failure", + "libc", +] + [[package]] name = "winapi" version = "0.2.8" @@ -4352,6 +4622,12 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "yaml-rust" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992" + [[package]] name = "zerocopy" version = "0.7.32" @@ -4367,8 +4643,8 @@ version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.79", + "quote 1.0.36", "syn 2.0.58", ] diff --git a/rust_snuba/Cargo.toml b/rust_snuba/Cargo.toml index ba0f3f319e..39c3f0653d 100644 --- a/rust_snuba/Cargo.toml +++ b/rust_snuba/Cargo.toml @@ -50,6 +50,7 @@ json-schema-diff = "0.1.7" serde_path_to_error = "0.1.15" hyper = "1.2.0" tokio-stream = "0.1.15" +cityhash = "0.1.1" data-encoding = "2.5.0" zstd = "0.12.3" serde_with = "3.8.1" diff --git a/rust_snuba/src/processors/mod.rs b/rust_snuba/src/processors/mod.rs index ce0e30328a..694ec48dd5 100644 --- a/rust_snuba/src/processors/mod.rs +++ b/rust_snuba/src/processors/mod.rs @@ -9,6 +9,7 @@ mod querylog; mod release_health_metrics; mod replays; mod spans; +mod spans_v2; mod utils; use crate::config::ProcessorConfig; @@ -55,6 +56,7 @@ define_processing_functions! { ("QuerylogProcessor", "snuba-queries", ProcessingFunctionType::ProcessingFunction(querylog::process_message)), ("ReplaysProcessor", "ingest-replay-events", ProcessingFunctionType::ProcessingFunction(replays::process_message)), ("SpansMessageProcessor", "snuba-spans", ProcessingFunctionType::ProcessingFunction(spans::process_message)), + ("SpansV2MessageProcessor", "snuba-spans", ProcessingFunctionType::ProcessingFunction(spans_v2::process_message)), ("MetricsSummariesMessageProcessor", "snuba-metrics-summaries", ProcessingFunctionType::ProcessingFunction(metrics_summaries::process_message)), ("OutcomesProcessor", "outcomes", ProcessingFunctionType::ProcessingFunction(outcomes::process_message)), ("GenericCountersMetricsProcessor", "snuba-generic-metrics", ProcessingFunctionType::ProcessingFunction(generic_metrics::process_counter_message)), diff --git a/rust_snuba/src/processors/spans.rs b/rust_snuba/src/processors/spans.rs index f74ab3462c..c8ceccec89 100644 --- a/rust_snuba/src/processors/spans.rs +++ b/rust_snuba/src/processors/spans.rs @@ -32,33 +32,33 @@ pub fn process_message( } #[derive(Debug, Default, Deserialize, JsonSchema)] -struct FromSpanMessage { - description: Option, - duration_ms: u32, - #[serde(alias = "end_timestamp_micro")] - end_timestamp_precise: Option, - event_id: Option, - exclusive_time_ms: f64, - is_segment: bool, - measurements: Option>, - parent_span_id: Option, - profile_id: Option, - project_id: u64, - received: f64, - retention_days: Option, - segment_id: Option, - sentry_tags: Option>, - span_id: String, +pub(crate) struct FromSpanMessage { + pub(crate) description: Option, + pub(crate) duration_ms: u32, + pub(crate) end_timestamp_precise: f64, + pub(crate) event_id: Option, + pub(crate) exclusive_time_ms: f64, + pub(crate) is_segment: bool, + pub(crate) measurements: Option>, + pub(crate) parent_span_id: Option, + pub(crate) profile_id: Option, + pub(crate) organization_id: u64, + pub(crate) project_id: u64, + pub(crate) received: f64, + pub(crate) retention_days: Option, + pub(crate) segment_id: Option, + pub(crate) sentry_tags: Option>, + pub(crate) span_id: String, #[serde(alias = "start_timestamp_micro")] - start_timestamp_precise: Option, - start_timestamp_ms: u64, - tags: Option>, - trace_id: Uuid, + pub(crate) start_timestamp_precise: Option, + pub(crate) start_timestamp_ms: u64, + pub(crate) tags: Option>, + pub(crate) trace_id: Uuid, } #[derive(Debug, Default, Deserialize, JsonSchema)] -struct FromMeasurementValue { - value: f64, +pub(crate) struct FromMeasurementValue { + pub(crate) value: f64, } #[derive(Debug, Default, Serialize)] @@ -153,7 +153,7 @@ impl TryFrom for Span { duration: from.duration_ms, end_ms: (end_timestamp_ms % 1000) as u16, end_timestamp: end_timestamp_ms / 1000, - end_timestamp_precise: (from.end_timestamp_precise.unwrap_or_default() * 1e6) as u64, + end_timestamp_precise: (from.end_timestamp_precise * 1e6) as u64, exclusive_time: from.exclusive_time_ms, group, is_segment: if from.is_segment { 1 } else { 0 }, diff --git a/rust_snuba/src/processors/spans_v2.rs b/rust_snuba/src/processors/spans_v2.rs new file mode 100644 index 0000000000..d3db2c7d01 --- /dev/null +++ b/rust_snuba/src/processors/spans_v2.rs @@ -0,0 +1,168 @@ +use anyhow::Context; +use chrono::DateTime; +use cityhash::cityhash_1::city_hash_64; +use serde::Serialize; +use std::collections::HashMap; +use uuid::Uuid; + +use rust_arroyo::backends::kafka::types::KafkaPayload; + +use crate::config::ProcessorConfig; +use crate::processors::spans::FromSpanMessage; +use crate::processors::utils::enforce_retention; +use crate::types::{InsertBatch, KafkaMessageMetadata}; + +pub fn process_message( + payload: KafkaPayload, + _metadata: KafkaMessageMetadata, + config: &ProcessorConfig, +) -> anyhow::Result { + let payload_bytes = payload.payload().context("Expected payload")?; + let msg: FromSpanMessage = serde_json::from_slice(payload_bytes)?; + + let origin_timestamp = DateTime::from_timestamp(msg.received as i64, 0); + let mut span: SpanV2 = msg.try_into()?; + + span.retention_days = Some(enforce_retention(span.retention_days, &config.env_config)); + + InsertBatch::from_rows([span], origin_timestamp) +} + +#[derive(Debug, Default, Serialize)] +struct SpanV2 { + organization_id: u64, + project_id: u64, + trace_id: Uuid, + span_id: u64, + #[serde(default)] + parent_span_id: u64, + segment_id: u64, //aka transaction ID + segment_name: String, //aka transaction name + is_segment: bool, //aka "is transaction" + _sort_timestamp: u32, + start_timestamp: u64, + end_timestamp: u64, + duration_ms: u32, + exclusive_time_ms: f64, + retention_days: Option, + name: String, //aka description + + sampling_factor: f64, + sampling_weight: f64, + sign: u8, //1 for additions, -1 for deletions - for this worker it should be 1 + + attr_str_0: HashMap, + attr_str_1: HashMap, + attr_str_2: HashMap, + attr_str_3: HashMap, + attr_str_4: HashMap, + attr_str_5: HashMap, + attr_str_6: HashMap, + attr_str_7: HashMap, + attr_str_8: HashMap, + attr_str_9: HashMap, + + attr_num_0: HashMap, + attr_num_1: HashMap, + attr_num_2: HashMap, + attr_num_3: HashMap, + attr_num_4: HashMap, + attr_num_5: HashMap, + attr_num_6: HashMap, + attr_num_7: HashMap, + attr_num_8: HashMap, + attr_num_9: HashMap, + + attr_bool_0: HashMap, + attr_bool_1: HashMap, + attr_bool_2: HashMap, + attr_bool_3: HashMap, + attr_bool_4: HashMap, + attr_bool_5: HashMap, + attr_bool_6: HashMap, + attr_bool_7: HashMap, + attr_bool_8: HashMap, + attr_bool_9: HashMap, +} + +impl TryFrom for SpanV2 { + type Error = anyhow::Error; + + fn try_from(from: FromSpanMessage) -> anyhow::Result { + let sentry_tags = from.sentry_tags.unwrap_or_default(); + let tags = from.tags.unwrap_or_default(); + let measurements = from.measurements.unwrap_or_default(); + + let mut res = Self { + organization_id: from.organization_id, + project_id: from.project_id, + trace_id: from.trace_id, + span_id: u64::from_str_radix(&from.span_id, 16).unwrap_or_default(), + parent_span_id: from + .parent_span_id + .map_or(0, |s| u64::from_str_radix(&s, 16).unwrap_or(0)), + segment_id: from + .segment_id + .map_or(0, |s| u64::from_str_radix(&s, 16).unwrap_or(0)), + segment_name: sentry_tags.get("transaction").cloned().unwrap_or_default(), + is_segment: from.is_segment, + _sort_timestamp: (from.start_timestamp_ms / 1000) as u32, + start_timestamp: from.start_timestamp_ms, + end_timestamp: from.start_timestamp_ms + from.duration_ms as u64, + duration_ms: from.duration_ms, + exclusive_time_ms: from.exclusive_time_ms, + retention_days: from.retention_days, + name: from.description.unwrap_or_default(), + + sampling_weight: 1., + sampling_factor: 1., + sign: 1, + + ..Default::default() + }; + + { + let mut attr_str_buckets = [ + &mut res.attr_str_0, + &mut res.attr_str_1, + &mut res.attr_str_2, + &mut res.attr_str_3, + &mut res.attr_str_4, + &mut res.attr_str_5, + &mut res.attr_str_6, + &mut res.attr_str_7, + &mut res.attr_str_8, + &mut res.attr_str_9, + ]; + + sentry_tags.iter().chain(tags.iter()).for_each(|(k, v)| { + attr_str_buckets[(city_hash_64(k.as_bytes()) as usize) % attr_str_buckets.len()] + .insert(k.clone(), v.clone()); + () + }); + } + + { + let mut attr_num_buckets = [ + &mut res.attr_num_0, + &mut res.attr_num_1, + &mut res.attr_num_2, + &mut res.attr_num_3, + &mut res.attr_num_4, + &mut res.attr_num_5, + &mut res.attr_num_6, + &mut res.attr_num_7, + &mut res.attr_num_8, + &mut res.attr_num_9, + ]; + + measurements.iter().for_each(|(k, v)| { + attr_num_buckets[(city_hash_64(k.as_bytes()) as usize) % attr_num_buckets.len()] + .insert(k.clone(), v.value); + () + }); + } + + Ok(res) + } +} diff --git a/snuba/datasets/processors/spans_v2_processor.py b/snuba/datasets/processors/spans_v2_processor.py new file mode 100644 index 0000000000..8babef7e85 --- /dev/null +++ b/snuba/datasets/processors/spans_v2_processor.py @@ -0,0 +1,6 @@ +from snuba.datasets.processors.rust_compat_processor import RustCompatProcessor + + +class SpansV2MessageProcessor(RustCompatProcessor): + def __init__(self) -> None: + super().__init__("SpansV2MessageProcessor") From 98cee83e43f4e344a2d9c828be9249cf93a85c39 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Thu, 18 Jul 2024 13:29:33 -0400 Subject: [PATCH 02/17] Add test --- ...ssors__spans_v2__tests__serialization.snap | 91 ++++++++++++++++++ rust_snuba/src/processors/spans_v2.rs | 94 ++++++++++++++++++- 2 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap new file mode 100644 index 0000000000..3ee4ecda77 --- /dev/null +++ b/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap @@ -0,0 +1,91 @@ +--- +source: src/processors/spans_v2.rs +expression: span +--- +{ + "organization_id": 1, + "project_id": 1, + "trace_id": "d099bf9a-d5a1-43cf-8f83-a98081d0ed3b", + "span_id": 9832388815107059821, + "parent_span_id": 0, + "segment_id": 9832388815107059821, + "segment_name": "/api/0/relays/projectconfigs/", + "is_segment": true, + "_sort_timestamp": 1721319572, + "start_timestamp": 1721319572616, + "end_timestamp": 1721319572768, + "duration_ms": 152, + "exclusive_time_ms": 0.228, + "retention_days": 90, + "name": "/api/0/relays/projectconfigs/", + "sampling_factor": 1.0, + "sampling_weight": 1.0, + "sign": 1, + "attr_str_0": { + "relay_id": "88888888-4444-4444-8444-cccccccccccc", + "op": "http.server", + "spans_over_limit": "False" + }, + "attr_str_1": { + "server_name": "D23CXQ4GK2.local" + }, + "attr_str_2": { + "status": "ok", + "environment": "development" + }, + "attr_str_3": { + "relay_no_cache": "False" + }, + "attr_str_4": { + "relay_protocol_version": "3", + "platform": "python", + "thread.id": "8522009600", + "sdk.version": "2.7.0" + }, + "attr_str_5": { + "status_code": "200", + "thread.name": "uWSGIWorker1Core0" + }, + "attr_str_6": { + "relay_endpoint_version": "3", + "release": "backend@24.7.0.dev0+c45b49caed1e5fcbf70097ab3f434b487c359b6b" + }, + "attr_str_7": { + "user": "ip:127.0.0.1" + }, + "attr_str_8": { + "transaction.method": "POST", + "trace.status": "ok", + "category": "http", + "transaction": "/api/0/relays/projectconfigs/", + "transaction.op": "http.server" + }, + "attr_str_9": { + "http.status_code": "200", + "sdk.name": "sentry.python.django", + "relay_use_post_or_schedule_rejected": "version", + "relay_use_post_or_schedule": "True" + }, + "attr_num_0": {}, + "attr_num_1": {}, + "attr_num_2": { + "num_of_spans": 50.0 + }, + "attr_num_3": {}, + "attr_num_4": {}, + "attr_num_5": {}, + "attr_num_6": {}, + "attr_num_7": {}, + "attr_num_8": {}, + "attr_num_9": {}, + "attr_bool_0": {}, + "attr_bool_1": {}, + "attr_bool_2": {}, + "attr_bool_3": {}, + "attr_bool_4": {}, + "attr_bool_5": {}, + "attr_bool_6": {}, + "attr_bool_7": {}, + "attr_bool_8": {}, + "attr_bool_9": {} +} diff --git a/rust_snuba/src/processors/spans_v2.rs b/rust_snuba/src/processors/spans_v2.rs index d3db2c7d01..0490dde7b4 100644 --- a/rust_snuba/src/processors/spans_v2.rs +++ b/rust_snuba/src/processors/spans_v2.rs @@ -1,8 +1,8 @@ use anyhow::Context; use chrono::DateTime; use cityhash::cityhash_1::city_hash_64; -use serde::Serialize; -use std::collections::HashMap; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap}; use uuid::Uuid; use rust_arroyo::backends::kafka::types::KafkaPayload; @@ -166,3 +166,93 @@ impl TryFrom for SpanV2 { Ok(res) } } + +#[cfg(test)] +mod tests { + use std::time::SystemTime; + + use super::*; + + const SPAN_KAFKA_MESSAGE: &str = r#" +{ + "description": "/api/0/relays/projectconfigs/", + "duration_ms": 152, + "event_id": "d826225de75d42d6b2f01b957d51f18f", + "exclusive_time_ms": 0.228, + "is_segment": true, + "data": { + "sentry.environment": "development", + "sentry.release": "backend@24.7.0.dev0+c45b49caed1e5fcbf70097ab3f434b487c359b6b", + "thread.name": "uWSGIWorker1Core0", + "thread.id": "8522009600", + "sentry.segment.name": "/api/0/relays/projectconfigs/", + "sentry.sdk.name": "sentry.python.django", + "sentry.sdk.version": "2.7.0" + }, + "measurements": { + "num_of_spans": { + "value": 50.0 + } + }, + "organization_id": 1, + "origin": "auto.http.django", + "project_id": 1, + "received": 1721319572.877828, + "retention_days": 90, + "segment_id": "8873a98879faf06d", + "sentry_tags": { + "category": "http", + "environment": "development", + "op": "http.server", + "platform": "python", + "release": "backend@24.7.0.dev0+c45b49caed1e5fcbf70097ab3f434b487c359b6b", + "sdk.name": "sentry.python.django", + "sdk.version": "2.7.0", + "status": "ok", + "status_code": "200", + "thread.id": "8522009600", + "thread.name": "uWSGIWorker1Core0", + "trace.status": "ok", + "transaction": "/api/0/relays/projectconfigs/", + "transaction.method": "POST", + "transaction.op": "http.server", + "user": "ip:127.0.0.1" + }, + "span_id": "8873a98879faf06d", + "tags": { + "http.status_code": "200", + "relay_endpoint_version": "3", + "relay_id": "88888888-4444-4444-8444-cccccccccccc", + "relay_no_cache": "False", + "relay_protocol_version": "3", + "relay_use_post_or_schedule": "True", + "relay_use_post_or_schedule_rejected": "version", + "server_name": "D23CXQ4GK2.local", + "spans_over_limit": "False" + }, + "trace_id": "d099bf9ad5a143cf8f83a98081d0ed3b", + "start_timestamp_ms": 1721319572616, + "start_timestamp_precise": 1721319572.616648, + "end_timestamp_precise": 1721319572.768806 +} + "#; + + #[test] + fn test_valid_span() { + let payload = KafkaPayload::new(None, None, Some(SPAN_KAFKA_MESSAGE.as_bytes().to_vec())); + let meta = KafkaMessageMetadata { + partition: 0, + offset: 1, + timestamp: DateTime::from(SystemTime::now()), + }; + process_message(payload, meta, &ProcessorConfig::default()) + .expect("The message should be processed"); + } + + #[test] + fn test_serialization() { + let msg: FromSpanMessage = serde_json::from_slice(SPAN_KAFKA_MESSAGE.as_bytes()).unwrap(); + let span: SpanV2 = msg.try_into().unwrap(); + insta::assert_json_snapshot!(span); + } +} From 7ed1366fe3ccbe77a61f8c527ca83a317eace9da Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Thu, 18 Jul 2024 14:05:55 -0400 Subject: [PATCH 03/17] More buckets, fix test flake --- ...ssors__spans_v2__tests__serialization.snap | 97 +++--- rust_snuba/src/processors/spans_v2.rs | 278 +++++++++++++++++- snuba/cli/devserver.py | 11 + 3 files changed, 335 insertions(+), 51 deletions(-) diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap index 3ee4ecda77..c7389fe93c 100644 --- a/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap +++ b/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap @@ -22,70 +22,71 @@ expression: span "sampling_weight": 1.0, "sign": 1, "attr_str_0": { - "relay_id": "88888888-4444-4444-8444-cccccccccccc", - "op": "http.server", "spans_over_limit": "False" }, - "attr_str_1": { - "server_name": "D23CXQ4GK2.local" - }, "attr_str_2": { - "status": "ok", - "environment": "development" - }, - "attr_str_3": { - "relay_no_cache": "False" + "status": "ok" }, "attr_str_4": { - "relay_protocol_version": "3", "platform": "python", - "thread.id": "8522009600", "sdk.version": "2.7.0" }, - "attr_str_5": { - "status_code": "200", - "thread.name": "uWSGIWorker1Core0" - }, "attr_str_6": { - "relay_endpoint_version": "3", "release": "backend@24.7.0.dev0+c45b49caed1e5fcbf70097ab3f434b487c359b6b" }, - "attr_str_7": { + "attr_str_9": { + "http.status_code": "200" + }, + "attr_str_10": { + "relay_id": "88888888-4444-4444-8444-cccccccccccc" + }, + "attr_str_17": { "user": "ip:127.0.0.1" }, - "attr_str_8": { - "transaction.method": "POST", + "attr_str_18": { "trace.status": "ok", + "transaction.method": "POST" + }, + "attr_str_20": { + "op": "http.server" + }, + "attr_str_23": { + "relay_no_cache": "False" + }, + "attr_str_25": { + "thread.name": "uWSGIWorker1Core0" + }, + "attr_str_26": { + "relay_endpoint_version": "3" + }, + "attr_str_28": { + "transaction": "/api/0/relays/projectconfigs/" + }, + "attr_str_29": { + "relay_use_post_or_schedule": "True", + "sdk.name": "sentry.python.django" + }, + "attr_str_39": { + "relay_use_post_or_schedule_rejected": "version" + }, + "attr_str_41": { + "server_name": "D23CXQ4GK2.local" + }, + "attr_str_42": { + "environment": "development" + }, + "attr_str_44": { + "relay_protocol_version": "3", + "thread.id": "8522009600" + }, + "attr_str_45": { + "status_code": "200" + }, + "attr_str_48": { "category": "http", - "transaction": "/api/0/relays/projectconfigs/", "transaction.op": "http.server" }, - "attr_str_9": { - "http.status_code": "200", - "sdk.name": "sentry.python.django", - "relay_use_post_or_schedule_rejected": "version", - "relay_use_post_or_schedule": "True" - }, - "attr_num_0": {}, - "attr_num_1": {}, - "attr_num_2": { + "attr_num_42": { "num_of_spans": 50.0 - }, - "attr_num_3": {}, - "attr_num_4": {}, - "attr_num_5": {}, - "attr_num_6": {}, - "attr_num_7": {}, - "attr_num_8": {}, - "attr_num_9": {}, - "attr_bool_0": {}, - "attr_bool_1": {}, - "attr_bool_2": {}, - "attr_bool_3": {}, - "attr_bool_4": {}, - "attr_bool_5": {}, - "attr_bool_6": {}, - "attr_bool_7": {}, - "attr_bool_8": {}, - "attr_bool_9": {} + } } diff --git a/rust_snuba/src/processors/spans_v2.rs b/rust_snuba/src/processors/spans_v2.rs index 0490dde7b4..0d240340dd 100644 --- a/rust_snuba/src/processors/spans_v2.rs +++ b/rust_snuba/src/processors/spans_v2.rs @@ -1,8 +1,8 @@ use anyhow::Context; use chrono::DateTime; use cityhash::cityhash_1::city_hash_64; -use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashMap}; +use serde::Serialize; +use std::collections::HashMap; use uuid::Uuid; use rust_arroyo::backends::kafka::types::KafkaPayload; @@ -51,37 +51,227 @@ struct SpanV2 { sampling_weight: f64, sign: u8, //1 for additions, -1 for deletions - for this worker it should be 1 + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_str_0: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_str_1: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_str_2: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_str_3: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_str_4: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_str_5: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_str_6: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_str_7: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_str_8: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_str_9: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_10: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_11: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_12: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_13: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_14: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_15: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_16: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_17: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_18: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_19: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_20: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_21: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_22: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_23: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_24: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_25: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_26: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_27: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_28: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_29: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_30: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_31: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_32: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_33: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_34: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_35: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_36: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_37: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_38: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_39: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_40: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_41: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_42: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_43: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_44: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_45: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_46: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_47: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_48: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_str_49: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_num_0: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_num_1: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_num_2: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_num_3: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_num_4: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_num_5: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_num_6: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_num_7: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_num_8: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_num_9: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_10: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_11: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_12: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_13: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_14: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_15: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_16: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_17: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_18: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_19: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_20: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_21: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_22: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_23: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_24: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_25: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_26: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_27: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_28: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_29: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_30: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_31: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_32: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_33: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_34: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_35: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_36: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_37: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_38: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_39: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_40: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_41: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_42: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_43: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_44: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_45: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_46: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_47: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_48: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_num_49: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_0: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_1: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_2: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_3: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_4: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_5: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_6: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_7: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_8: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_9: HashMap, } @@ -133,6 +323,46 @@ impl TryFrom for SpanV2 { &mut res.attr_str_7, &mut res.attr_str_8, &mut res.attr_str_9, + &mut res.attr_str_10, + &mut res.attr_str_11, + &mut res.attr_str_12, + &mut res.attr_str_13, + &mut res.attr_str_14, + &mut res.attr_str_15, + &mut res.attr_str_16, + &mut res.attr_str_17, + &mut res.attr_str_18, + &mut res.attr_str_19, + &mut res.attr_str_20, + &mut res.attr_str_21, + &mut res.attr_str_22, + &mut res.attr_str_23, + &mut res.attr_str_24, + &mut res.attr_str_25, + &mut res.attr_str_26, + &mut res.attr_str_27, + &mut res.attr_str_28, + &mut res.attr_str_29, + &mut res.attr_str_30, + &mut res.attr_str_31, + &mut res.attr_str_32, + &mut res.attr_str_33, + &mut res.attr_str_34, + &mut res.attr_str_35, + &mut res.attr_str_36, + &mut res.attr_str_37, + &mut res.attr_str_38, + &mut res.attr_str_39, + &mut res.attr_str_40, + &mut res.attr_str_41, + &mut res.attr_str_42, + &mut res.attr_str_43, + &mut res.attr_str_44, + &mut res.attr_str_45, + &mut res.attr_str_46, + &mut res.attr_str_47, + &mut res.attr_str_48, + &mut res.attr_str_49, ]; sentry_tags.iter().chain(tags.iter()).for_each(|(k, v)| { @@ -154,6 +384,46 @@ impl TryFrom for SpanV2 { &mut res.attr_num_7, &mut res.attr_num_8, &mut res.attr_num_9, + &mut res.attr_num_10, + &mut res.attr_num_11, + &mut res.attr_num_12, + &mut res.attr_num_13, + &mut res.attr_num_14, + &mut res.attr_num_15, + &mut res.attr_num_16, + &mut res.attr_num_17, + &mut res.attr_num_18, + &mut res.attr_num_19, + &mut res.attr_num_20, + &mut res.attr_num_21, + &mut res.attr_num_22, + &mut res.attr_num_23, + &mut res.attr_num_24, + &mut res.attr_num_25, + &mut res.attr_num_26, + &mut res.attr_num_27, + &mut res.attr_num_28, + &mut res.attr_num_29, + &mut res.attr_num_30, + &mut res.attr_num_31, + &mut res.attr_num_32, + &mut res.attr_num_33, + &mut res.attr_num_34, + &mut res.attr_num_35, + &mut res.attr_num_36, + &mut res.attr_num_37, + &mut res.attr_num_38, + &mut res.attr_num_39, + &mut res.attr_num_40, + &mut res.attr_num_41, + &mut res.attr_num_42, + &mut res.attr_num_43, + &mut res.attr_num_44, + &mut res.attr_num_45, + &mut res.attr_num_46, + &mut res.attr_num_47, + &mut res.attr_num_48, + &mut res.attr_num_49, ]; measurements.iter().for_each(|(k, v)| { @@ -253,6 +523,8 @@ mod tests { fn test_serialization() { let msg: FromSpanMessage = serde_json::from_slice(SPAN_KAFKA_MESSAGE.as_bytes()).unwrap(); let span: SpanV2 = msg.try_into().unwrap(); - insta::assert_json_snapshot!(span); + insta::with_settings!({sort_maps => true}, { + insta::assert_json_snapshot!(span) + }); } } diff --git a/snuba/cli/devserver.py b/snuba/cli/devserver.py index 9264129baf..7a800d7016 100644 --- a/snuba/cli/devserver.py +++ b/snuba/cli/devserver.py @@ -107,6 +107,17 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: *COMMON_RUST_CONSUMER_DEV_OPTIONS, ], ), + ( + "spans-v2-consumer", + [ + "snuba", + "rust-consumer", + "--storage=spans_v2", + "--consumer-group=spans_v2_group", + "--use-rust-processor", + *COMMON_RUST_CONSUMER_DEV_OPTIONS, + ], + ), ] if settings.SEPARATE_SCHEDULER_EXECUTOR_SUBSCRIPTIONS_DEV: From 2c04415ec645a49dd7b00b5a1db309795807befb Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Thu, 18 Jul 2024 14:10:30 -0400 Subject: [PATCH 04/17] lint --- rust_snuba/src/processors/spans_v2.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust_snuba/src/processors/spans_v2.rs b/rust_snuba/src/processors/spans_v2.rs index 0d240340dd..2a53462ff3 100644 --- a/rust_snuba/src/processors/spans_v2.rs +++ b/rust_snuba/src/processors/spans_v2.rs @@ -368,7 +368,6 @@ impl TryFrom for SpanV2 { sentry_tags.iter().chain(tags.iter()).for_each(|(k, v)| { attr_str_buckets[(city_hash_64(k.as_bytes()) as usize) % attr_str_buckets.len()] .insert(k.clone(), v.clone()); - () }); } @@ -429,7 +428,6 @@ impl TryFrom for SpanV2 { measurements.iter().for_each(|(k, v)| { attr_num_buckets[(city_hash_64(k.as_bytes()) as usize) % attr_num_buckets.len()] .insert(k.clone(), v.value); - () }); } From 9ef0f9145330e00d69403c22a82b416859deae8c Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Thu, 18 Jul 2024 14:44:10 -0400 Subject: [PATCH 05/17] Remove cityhash --- rust_snuba/Cargo.lock | 418 +++--------------- rust_snuba/Cargo.toml | 1 - ...ssors__spans_v2__tests__serialization.snap | 86 ++-- rust_snuba/src/processors/spans_v2.rs | 11 +- 4 files changed, 121 insertions(+), 395 deletions(-) diff --git a/rust_snuba/Cargo.lock b/rust_snuba/Cargo.lock index 3c1cec3adf..b49c6bfe8e 100644 --- a/rust_snuba/Cargo.lock +++ b/rust_snuba/Cargo.lock @@ -27,7 +27,7 @@ dependencies = [ "getrandom 0.2.14", "once_cell", "serde", - "version_check 0.9.4", + "version_check", "zerocopy", ] @@ -67,15 +67,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "anstream" version = "0.6.13" @@ -158,7 +149,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" dependencies = [ - "quote 1.0.36", + "quote", "syn 1.0.109", ] @@ -358,8 +349,8 @@ version = "0.1.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] @@ -369,17 +360,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi 0.3.9", -] - [[package]] name = "autocfg" version = "1.2.0" @@ -434,29 +414,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.49.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c07087f3d5731bf3fb375a81841b99597e25dc11bd3bc72d16d43adf6624a6e" -dependencies = [ - "bitflags 1.3.2", - "cexpr", - "cfg-if 0.1.10", - "clang-sys", - "clap 2.34.0", - "env_logger 0.6.2", - "fxhash", - "lazy_static", - "log", - "peeking_take_while", - "proc-macro2 0.4.30", - "quote 0.6.13", - "regex", - "shlex", - "which", -] - [[package]] name = "bit-set" version = "0.5.3" @@ -521,12 +478,6 @@ version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.6.0" @@ -558,15 +509,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cexpr" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d" -dependencies = [ - "nom 4.2.3", -] - [[package]] name = "cfg-if" version = "0.1.10" @@ -627,46 +569,6 @@ dependencies = [ "half", ] -[[package]] -name = "cityhash" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7ecb82e5b6394ea8408bc8f454d50c75f339ca89f9af90a6ff8694e6a1c63da" -dependencies = [ - "bindgen", - "cc", - "clap 2.34.0", - "libc", -] - -[[package]] -name = "clang-sys" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853" -dependencies = [ - "glob", - "libc", - "libloading", -] - -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "ansi_term", - "atty", - "bitflags 1.3.2", - "strsim 0.8.0", - "term_size", - "textwrap", - "unicode-width", - "vec_map", - "yaml-rust", -] - [[package]] name = "clap" version = "4.5.4" @@ -686,7 +588,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.1", + "strsim", ] [[package]] @@ -696,8 +598,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ "heck 0.5.0", - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] @@ -797,7 +699,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.4", + "clap", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -880,7 +782,7 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ - "quote 1.0.36", + "quote", "syn 1.0.109", ] @@ -912,9 +814,9 @@ checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.79", - "quote 1.0.36", - "strsim 0.11.1", + "proc-macro2", + "quote", + "strsim", "syn 2.0.58", ] @@ -925,7 +827,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ "darling_core", - "quote 1.0.36", + "quote", "syn 2.0.58", ] @@ -951,8 +853,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a3767f826efbbe5a5ae093920b58b43b01734202be697e1354914e862e8e704" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "semver", "syn 2.0.58", ] @@ -1034,26 +936,13 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "env_logger" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" -dependencies = [ - "atty", - "humantime 1.3.0", - "log", - "regex", - "termcolor", -] - [[package]] name = "env_logger" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ - "humantime 2.1.0", + "humantime", "is-terminal", "log", "regex", @@ -1135,15 +1024,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "failure" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" -dependencies = [ - "backtrace", -] - [[package]] name = "fancy-regex" version = "0.11.0" @@ -1325,8 +1205,8 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] @@ -1360,15 +1240,6 @@ dependencies = [ "slab", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -1376,7 +1247,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", - "version_check 0.9.4", + "version_check", ] [[package]] @@ -1409,12 +1280,6 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - [[package]] name = "gloo-timers" version = "0.2.6" @@ -1493,15 +1358,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.9" @@ -1608,15 +1464,6 @@ dependencies = [ "url", ] -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - [[package]] name = "humantime" version = "2.1.0" @@ -1778,7 +1625,7 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", "windows-sys 0.48.0", ] @@ -1823,7 +1670,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", "windows-sys 0.52.0", ] @@ -1834,7 +1681,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -1902,7 +1749,7 @@ dependencies = [ "anyhow", "base64 0.21.7", "bytecount", - "clap 4.5.4", + "clap", "fancy-regex", "fraction", "getrandom 0.2.14", @@ -1959,7 +1806,7 @@ dependencies = [ "string_cache", "term", "tiny-keccak", - "unicode-xid 0.2.4", + "unicode-xid", "walkdir", ] @@ -1990,16 +1837,6 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" -[[package]] -name = "libloading" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" -dependencies = [ - "cc", - "winapi 0.3.9", -] - [[package]] name = "libredox" version = "0.1.3" @@ -2205,16 +2042,6 @@ dependencies = [ "libc", ] -[[package]] -name = "nom" -version = "4.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" -dependencies = [ - "memchr", - "version_check 0.1.5", -] - [[package]] name = "nom" version = "7.1.3" @@ -2328,7 +2155,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", ] @@ -2348,8 +2175,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -2395,8 +2222,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] @@ -2479,12 +2306,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "percent-encoding" version = "2.3.1" @@ -2520,8 +2341,8 @@ checksum = "c35eeed0a3fab112f75165fdc026b3913f4183133f19b49be773ac9ea966e8bd" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] @@ -2642,7 +2463,7 @@ checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" dependencies = [ "cfg-if 1.0.0", "concurrent-queue", - "hermit-abi 0.3.9", + "hermit-abi", "pin-project-lite", "rustix 0.38.32", "tracing", @@ -2673,7 +2494,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" dependencies = [ - "proc-macro2 1.0.79", + "proc-macro2", "syn 2.0.58", ] @@ -2687,15 +2508,6 @@ dependencies = [ "toml_edit", ] -[[package]] -name = "proc-macro2" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid 0.1.0", -] - [[package]] name = "proc-macro2" version = "1.0.79" @@ -2765,9 +2577,9 @@ version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d39c55dab3fc5a4b25bbd1ac10a2da452c4aca13bb450f22818a002e29648d" dependencies = [ - "proc-macro2 1.0.79", + "proc-macro2", "pyo3-macros-backend", - "quote 1.0.36", + "quote", "syn 1.0.109", ] @@ -2777,33 +2589,18 @@ version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97daff08a4c48320587b5224cc98d609e3c27b6d437315bd40b605c98eeb5918" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 1.0.109", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - -[[package]] -name = "quote" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] - [[package]] name = "quote" version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ - "proc-macro2 1.0.79", + "proc-macro2", ] [[package]] @@ -3081,7 +2878,6 @@ dependencies = [ "anyhow", "cadence", "chrono", - "cityhash", "criterion", "ctrlc", "data-encoding", @@ -3222,8 +3018,8 @@ version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "serde_derive_internals", "syn 1.0.109", ] @@ -3430,8 +3226,8 @@ version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] @@ -3441,8 +3237,8 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -3483,8 +3279,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a00ffd23fd882d096f09fcaae2a9de8329a328628e86027e049ee051dc1621f" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "serde", "syn 2.0.58", ] @@ -3526,8 +3322,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" dependencies = [ "darling", - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] @@ -3564,12 +3360,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shlex" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" - [[package]] name = "signal-hook" version = "0.3.17" @@ -3644,9 +3434,9 @@ checksum = "793ec303cef342e5a80e717016050673430851d5e3159ba8275c0f801cc83c11" dependencies = [ "anyhow", "cadence", - "clap 4.5.4", + "clap", "crc32fast", - "env_logger 0.10.2", + "env_logger", "log", "rand 0.8.5", "sentry", @@ -3669,12 +3459,6 @@ dependencies = [ "precomputed-hash", ] -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - [[package]] name = "strsim" version = "0.11.1" @@ -3687,8 +3471,8 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "unicode-ident", ] @@ -3698,8 +3482,8 @@ version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "unicode-ident", ] @@ -3759,16 +3543,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "term_size" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9" -dependencies = [ - "libc", - "winapi 0.3.9", -] - [[package]] name = "termcolor" version = "1.4.1" @@ -3778,16 +3552,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "term_size", - "unicode-width", -] - [[package]] name = "thiserror" version = "1.0.58" @@ -3803,8 +3567,8 @@ version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] @@ -3928,8 +3692,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] @@ -4008,8 +3772,8 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] @@ -4095,8 +3859,8 @@ checksum = "95e32f38493804f88e2dc7a5412eccd872ea5452b4db9b0a77de4df180f2a87e" dependencies = [ "heck 0.4.1", "log", - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "regress 0.8.0", "schemars", "serde_json", @@ -4111,8 +3875,8 @@ version = "0.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc09508b72f63d521d68e42c7f172c7416d67986df44b3c7d1f7f9963948ed32" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "schemars", "serde", "serde_json", @@ -4157,18 +3921,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-width" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" - -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - [[package]] name = "unicode-xid" version = "0.2.4" @@ -4246,18 +3998,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - -[[package]] -name = "version_check" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" - [[package]] name = "version_check" version = "0.9.4" @@ -4329,8 +4069,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", "wasm-bindgen-shared", ] @@ -4353,7 +4093,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ - "quote 1.0.36", + "quote", "wasm-bindgen-macro-support", ] @@ -4363,8 +4103,8 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -4399,16 +4139,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "which" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164" -dependencies = [ - "failure", - "libc", -] - [[package]] name = "winapi" version = "0.2.8" @@ -4622,12 +4352,6 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "yaml-rust" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992" - [[package]] name = "zerocopy" version = "0.7.32" @@ -4643,8 +4367,8 @@ version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.36", + "proc-macro2", + "quote", "syn 2.0.58", ] diff --git a/rust_snuba/Cargo.toml b/rust_snuba/Cargo.toml index 39c3f0653d..ba0f3f319e 100644 --- a/rust_snuba/Cargo.toml +++ b/rust_snuba/Cargo.toml @@ -50,7 +50,6 @@ json-schema-diff = "0.1.7" serde_path_to_error = "0.1.15" hyper = "1.2.0" tokio-stream = "0.1.15" -cityhash = "0.1.1" data-encoding = "2.5.0" zstd = "0.12.3" serde_with = "3.8.1" diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap index c7389fe93c..8010e3c7c5 100644 --- a/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap +++ b/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap @@ -21,72 +21,70 @@ expression: span "sampling_factor": 1.0, "sampling_weight": 1.0, "sign": 1, - "attr_str_0": { - "spans_over_limit": "False" - }, - "attr_str_2": { - "status": "ok" + "attr_str_3": { + "server_name": "D23CXQ4GK2.local" }, - "attr_str_4": { - "platform": "python", - "sdk.version": "2.7.0" + "attr_str_7": { + "relay_use_post_or_schedule_rejected": "version" }, - "attr_str_6": { - "release": "backend@24.7.0.dev0+c45b49caed1e5fcbf70097ab3f434b487c359b6b" + "attr_str_10": { + "transaction": "/api/0/relays/projectconfigs/" }, - "attr_str_9": { - "http.status_code": "200" + "attr_str_11": { + "thread.id": "8522009600" }, - "attr_str_10": { - "relay_id": "88888888-4444-4444-8444-cccccccccccc" + "attr_str_12": { + "relay_use_post_or_schedule": "True" }, "attr_str_17": { - "user": "ip:127.0.0.1" + "release": "backend@24.7.0.dev0+c45b49caed1e5fcbf70097ab3f434b487c359b6b" }, - "attr_str_18": { - "trace.status": "ok", - "transaction.method": "POST" + "attr_str_19": { + "environment": "development", + "platform": "python" }, - "attr_str_20": { - "op": "http.server" + "attr_str_21": { + "relay_endpoint_version": "3" }, "attr_str_23": { - "relay_no_cache": "False" - }, - "attr_str_25": { - "thread.name": "uWSGIWorker1Core0" + "status_code": "200" }, - "attr_str_26": { - "relay_endpoint_version": "3" + "attr_str_24": { + "http.status_code": "200" }, - "attr_str_28": { - "transaction": "/api/0/relays/projectconfigs/" + "attr_str_25": { + "user": "ip:127.0.0.1" }, - "attr_str_29": { - "relay_use_post_or_schedule": "True", - "sdk.name": "sentry.python.django" + "attr_str_30": { + "sdk.version": "2.7.0", + "trace.status": "ok" }, - "attr_str_39": { - "relay_use_post_or_schedule_rejected": "version" + "attr_str_31": { + "category": "http", + "thread.name": "uWSGIWorker1Core0" }, - "attr_str_41": { - "server_name": "D23CXQ4GK2.local" + "attr_str_33": { + "transaction.method": "POST" }, - "attr_str_42": { - "environment": "development" + "attr_str_34": { + "relay_id": "88888888-4444-4444-8444-cccccccccccc", + "spans_over_limit": "False", + "status": "ok" }, - "attr_str_44": { - "relay_protocol_version": "3", - "thread.id": "8522009600" + "attr_str_38": { + "relay_no_cache": "False", + "sdk.name": "sentry.python.django" }, - "attr_str_45": { - "status_code": "200" + "attr_str_40": { + "op": "http.server" }, "attr_str_48": { - "category": "http", "transaction.op": "http.server" }, - "attr_num_42": { + "attr_str_49": { + "relay_protocol_version": "3" + }, + "attr_num_17": { "num_of_spans": 50.0 } } diff --git a/rust_snuba/src/processors/spans_v2.rs b/rust_snuba/src/processors/spans_v2.rs index 2a53462ff3..d3c09b302a 100644 --- a/rust_snuba/src/processors/spans_v2.rs +++ b/rust_snuba/src/processors/spans_v2.rs @@ -1,6 +1,6 @@ use anyhow::Context; use chrono::DateTime; -use cityhash::cityhash_1::city_hash_64; +use md5; use serde::Serialize; use std::collections::HashMap; use uuid::Uuid; @@ -275,6 +275,11 @@ struct SpanV2 { attr_bool_9: HashMap, } +fn half_md5(input: &[u8]) -> u64 { + let full_hash = md5::compute(input).0; + u64::from_be_bytes(full_hash[0..8].try_into().expect("should not happen")) +} + impl TryFrom for SpanV2 { type Error = anyhow::Error; @@ -366,7 +371,7 @@ impl TryFrom for SpanV2 { ]; sentry_tags.iter().chain(tags.iter()).for_each(|(k, v)| { - attr_str_buckets[(city_hash_64(k.as_bytes()) as usize) % attr_str_buckets.len()] + attr_str_buckets[(half_md5(k.as_bytes()) as usize) % attr_str_buckets.len()] .insert(k.clone(), v.clone()); }); } @@ -426,7 +431,7 @@ impl TryFrom for SpanV2 { ]; measurements.iter().for_each(|(k, v)| { - attr_num_buckets[(city_hash_64(k.as_bytes()) as usize) % attr_num_buckets.len()] + attr_num_buckets[(half_md5(k.as_bytes()) as usize) % attr_num_buckets.len()] .insert(k.clone(), v.value); }); } From d8268e22cf7e5b5e96a47c7ee8b7512099f64752 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Thu, 18 Jul 2024 14:49:53 -0400 Subject: [PATCH 06/17] Add test for half_md5 --- rust_snuba/src/processors/spans_v2.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust_snuba/src/processors/spans_v2.rs b/rust_snuba/src/processors/spans_v2.rs index d3c09b302a..d88eda1829 100644 --- a/rust_snuba/src/processors/spans_v2.rs +++ b/rust_snuba/src/processors/spans_v2.rs @@ -510,6 +510,12 @@ mod tests { } "#; + #[test] + fn test_half_md5() { + //select halfMD5('test') == 688887797400064883 + assert_eq!(half_md5("test".as_bytes()), 688887797400064883) + } + #[test] fn test_valid_span() { let payload = KafkaPayload::new(None, None, Some(SPAN_KAFKA_MESSAGE.as_bytes().to_vec())); From 9165914905dc55f1360107d41a934648b8ef4b81 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Thu, 18 Jul 2024 15:38:58 -0400 Subject: [PATCH 07/17] Lint + test --- rust_snuba/src/processors/spans_v2.rs | 1 - tests/test_spans_api.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/rust_snuba/src/processors/spans_v2.rs b/rust_snuba/src/processors/spans_v2.rs index d88eda1829..741c7537e4 100644 --- a/rust_snuba/src/processors/spans_v2.rs +++ b/rust_snuba/src/processors/spans_v2.rs @@ -1,6 +1,5 @@ use anyhow::Context; use chrono::DateTime; -use md5; use serde::Serialize; use std::collections::HashMap; use uuid::Uuid; diff --git a/tests/test_spans_api.py b/tests/test_spans_api.py index 3c243b3b28..c6b2f08fc9 100644 --- a/tests/test_spans_api.py +++ b/tests/test_spans_api.py @@ -83,6 +83,7 @@ def generate_fizzbuzz_events(self) -> None: .process_message( { "project_id": p, + "organization_id": 1, "event_id": uuid.uuid4().hex, "deleted": 0, "is_segment": False, From 3ea6ce9faa37868b968c31db2c1a3b41a1e1a16c Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Mon, 22 Jul 2024 10:55:48 -0400 Subject: [PATCH 08/17] fix test --- tests/datasets/test_spans_payloads.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/datasets/test_spans_payloads.py b/tests/datasets/test_spans_payloads.py index bd6d100aae..17a41a3478 100644 --- a/tests/datasets/test_spans_payloads.py +++ b/tests/datasets/test_spans_payloads.py @@ -67,6 +67,7 @@ "partition": 2, "platform": "", "profile_id": None, + "organization_id": 1, "project_id": project_id, "retention_days": 90, "segment_id": 1311768467284833366, From 92e559ac2d23106ae93ab567746a3e89bdf84bc5 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Mon, 22 Jul 2024 11:28:02 -0400 Subject: [PATCH 09/17] fix test? --- tests/datasets/test_spans_payloads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/datasets/test_spans_payloads.py b/tests/datasets/test_spans_payloads.py index 17a41a3478..7578dc9aa6 100644 --- a/tests/datasets/test_spans_payloads.py +++ b/tests/datasets/test_spans_payloads.py @@ -30,6 +30,7 @@ "exclusive_time_ms": 1234567890123, "is_segment": True, "project_id": project_id, + "organization_id": 1, "received": received, "retention_days": 90, "segment_id": "1234567890123456", @@ -67,7 +68,6 @@ "partition": 2, "platform": "", "profile_id": None, - "organization_id": 1, "project_id": project_id, "retention_days": 90, "segment_id": 1311768467284833366, From 3f6bebb3191762f267b65f0c83fac9eccda0a05a Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Mon, 22 Jul 2024 11:33:41 -0400 Subject: [PATCH 10/17] Snapshot test --- ...essor-snuba-spans__1__basic_span.json.snap | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-spans-SpansV2MessageProcessor-snuba-spans__1__basic_span.json.snap diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-spans-SpansV2MessageProcessor-snuba-spans__1__basic_span.json.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-spans-SpansV2MessageProcessor-snuba-spans__1__basic_span.json.snap new file mode 100644 index 0000000000..5b719ba494 --- /dev/null +++ b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-spans-SpansV2MessageProcessor-snuba-spans__1__basic_span.json.snap @@ -0,0 +1,73 @@ +--- +source: src/processors/mod.rs +description: "{\n \"event_id\": \"dcc403b73ef548648188bbfa6012e9dc\",\n \"organization_id\": 69,\n \"project_id\": 1,\n \"trace_id\": \"deadbeefdeadbeefdeadbeefdeadbeef\",\n \"span_id\": \"deadbeefdeadbeef\",\n \"parent_span_id\": \"deadbeefdeadbeef\",\n \"segment_id\": \"deadbeefdeadbeef\",\n \"duration_ms\": 1000,\n \"exclusive_time_ms\": 1000,\n \"is_segment\": false,\n \"profile_id\": \"deadbeefdeadbeefdeadbeefdeadbeef\",\n \"received\": 1715868485.381,\n \"retention_days\": 90,\n \"start_timestamp_ms\": 1715868485371,\n \"start_timestamp_precise\": 1715868485.370551,\n \"end_timestamp_precise\": 1715868486.370551,\n \"tags\": {\n \"tag1\": \"value1\",\n \"tag2\": \"123\",\n \"tag3\": \"True\"\n },\n \"sentry_tags\": {\n \"http.method\": \"GET\",\n \"action\": \"GET\",\n \"domain\": \"targetdomain.tld:targetport\",\n \"module\": \"http\",\n \"group\": \"deadbeefdeadbeef\",\n \"status\": \"ok\",\n \"system\": \"python\",\n \"status_code\": \"200\",\n \"transaction\": \"/organizations/:orgId/issues/\",\n \"transaction.op\": \"navigation\",\n \"op\": \"http.client\",\n \"transaction.method\": \"GET\"\n },\n \"measurements\": {\n \"http.response_content_length\": {\n \"value\": 100.0,\n \"unit\": \"byte\"\n }\n },\n \"_metrics_summary\": {\n \"c:sentry.events.outcomes@none\": [\n {\n \"count\": 1,\n \"max\": 1.0,\n \"min\": 1.0,\n \"sum\": 1.0,\n \"tags\": {\n \"category\": \"error\",\n \"environment\": \"unknown\",\n \"event_type\": \"error\",\n \"outcome\": \"accepted\",\n \"release\": \"backend@2af74c237fbd61489a1ccc46650f4f85befaf8b8\",\n \"topic\": \"outcomes-billing\",\n \"transaction\": \"sentry.tasks.store.save_event\"\n }\n }\n ],\n \"c:sentry.events.post_save.normalize.errors@none\": [\n {\n \"count\": 1,\n \"max\": 0.0,\n \"min\": 0.0,\n \"sum\": 0.0,\n \"tags\": {\n \"environment\": \"unknown\",\n \"event_type\": \"error\",\n \"from_relay\": \"False\",\n \"release\": \"backend@2af74c237fbd61489a1ccc46650f4f85befaf8b8\",\n \"transaction\": \"sentry.tasks.store.save_event\"\n }\n }\n ]\n }\n}\n" +expression: snapshot_payload +--- +[ + { + "_sort_timestamp": 1715868485, + "attr_num_12": { + "http.response_content_length": 100.0 + }, + "attr_str_1": { + "tag1": "value1", + "tag3": "True" + }, + "attr_str_10": { + "transaction": "/organizations/:orgId/issues/" + }, + "attr_str_12": { + "system": "python" + }, + "attr_str_16": { + "domain": "targetdomain.tld:targetport" + }, + "attr_str_17": { + "tag2": "123" + }, + "attr_str_21": { + "http.method": "GET" + }, + "attr_str_23": { + "status_code": "200" + }, + "attr_str_30": { + "group": "deadbeefdeadbeef" + }, + "attr_str_32": { + "module": "http" + }, + "attr_str_33": { + "transaction.method": "GET" + }, + "attr_str_34": { + "status": "ok" + }, + "attr_str_40": { + "op": "http.client" + }, + "attr_str_48": { + "transaction.op": "navigation" + }, + "attr_str_7": { + "action": "GET" + }, + "duration_ms": 1000, + "end_timestamp": 1715868486371, + "exclusive_time_ms": 1000.0, + "is_segment": false, + "name": "", + "organization_id": 69, + "parent_span_id": 16045690984833335023, + "project_id": 1, + "retention_days": 90, + "sampling_factor": 1.0, + "sampling_weight": 1.0, + "segment_id": 16045690984833335023, + "segment_name": "/organizations/:orgId/issues/", + "sign": 1, + "span_id": 16045690984833335023, + "start_timestamp": 1715868485371, + "trace_id": "deadbeef-dead-beef-dead-beefdeadbeef" + } +] From a9e22bfbe81950d111bdef3c04e9f14b8f006dea Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Mon, 22 Jul 2024 13:20:08 -0400 Subject: [PATCH 11/17] e to the a to the p --- .../src/processors/{spans_v2.rs => eap_spans.rs} | 13 +++++++------ rust_snuba/src/processors/mod.rs | 4 ++-- ..._processors__spans_v2__tests__serialization.snap | 2 +- snuba/cli/devserver.py | 4 ++-- snuba/datasets/processors/spans_v2_processor.py | 4 ++-- tests/datasets/test_spans_processor.py | 1 + 6 files changed, 15 insertions(+), 13 deletions(-) rename rust_snuba/src/processors/{spans_v2.rs => eap_spans.rs} (98%) diff --git a/rust_snuba/src/processors/spans_v2.rs b/rust_snuba/src/processors/eap_spans.rs similarity index 98% rename from rust_snuba/src/processors/spans_v2.rs rename to rust_snuba/src/processors/eap_spans.rs index 741c7537e4..d16e2780d2 100644 --- a/rust_snuba/src/processors/spans_v2.rs +++ b/rust_snuba/src/processors/eap_spans.rs @@ -20,7 +20,7 @@ pub fn process_message( let msg: FromSpanMessage = serde_json::from_slice(payload_bytes)?; let origin_timestamp = DateTime::from_timestamp(msg.received as i64, 0); - let mut span: SpanV2 = msg.try_into()?; + let mut span: EAPSpan = msg.try_into()?; span.retention_days = Some(enforce_retention(span.retention_days, &config.env_config)); @@ -28,7 +28,8 @@ pub fn process_message( } #[derive(Debug, Default, Serialize)] -struct SpanV2 { +struct EAPSpan { + // the span object for the new "events analytics platform" organization_id: u64, project_id: u64, trace_id: Uuid, @@ -279,10 +280,10 @@ fn half_md5(input: &[u8]) -> u64 { u64::from_be_bytes(full_hash[0..8].try_into().expect("should not happen")) } -impl TryFrom for SpanV2 { +impl From for EAPSpan { type Error = anyhow::Error; - fn try_from(from: FromSpanMessage) -> anyhow::Result { + fn from(from: FromSpanMessage) -> EAPSpan { let sentry_tags = from.sentry_tags.unwrap_or_default(); let tags = from.tags.unwrap_or_default(); let measurements = from.measurements.unwrap_or_default(); @@ -435,7 +436,7 @@ impl TryFrom for SpanV2 { }); } - Ok(res) + res } } @@ -530,7 +531,7 @@ mod tests { #[test] fn test_serialization() { let msg: FromSpanMessage = serde_json::from_slice(SPAN_KAFKA_MESSAGE.as_bytes()).unwrap(); - let span: SpanV2 = msg.try_into().unwrap(); + let span: EAPSpan = msg.try_into().unwrap(); insta::with_settings!({sort_maps => true}, { insta::assert_json_snapshot!(span) }); diff --git a/rust_snuba/src/processors/mod.rs b/rust_snuba/src/processors/mod.rs index 694ec48dd5..919da00cb5 100644 --- a/rust_snuba/src/processors/mod.rs +++ b/rust_snuba/src/processors/mod.rs @@ -1,3 +1,4 @@ +mod eap_spans; mod errors; mod functions; mod generic_metrics; @@ -9,7 +10,6 @@ mod querylog; mod release_health_metrics; mod replays; mod spans; -mod spans_v2; mod utils; use crate::config::ProcessorConfig; @@ -56,7 +56,7 @@ define_processing_functions! { ("QuerylogProcessor", "snuba-queries", ProcessingFunctionType::ProcessingFunction(querylog::process_message)), ("ReplaysProcessor", "ingest-replay-events", ProcessingFunctionType::ProcessingFunction(replays::process_message)), ("SpansMessageProcessor", "snuba-spans", ProcessingFunctionType::ProcessingFunction(spans::process_message)), - ("SpansV2MessageProcessor", "snuba-spans", ProcessingFunctionType::ProcessingFunction(spans_v2::process_message)), + ("EAPSpansMessageProcessor", "snuba-spans", ProcessingFunctionType::ProcessingFunction(eap_spans::process_message)), ("MetricsSummariesMessageProcessor", "snuba-metrics-summaries", ProcessingFunctionType::ProcessingFunction(metrics_summaries::process_message)), ("OutcomesProcessor", "outcomes", ProcessingFunctionType::ProcessingFunction(outcomes::process_message)), ("GenericCountersMetricsProcessor", "snuba-generic-metrics", ProcessingFunctionType::ProcessingFunction(generic_metrics::process_counter_message)), diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap index 8010e3c7c5..16cc974b51 100644 --- a/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap +++ b/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap @@ -1,5 +1,5 @@ --- -source: src/processors/spans_v2.rs +source: src/processors/eap_spans.rs expression: span --- { diff --git a/snuba/cli/devserver.py b/snuba/cli/devserver.py index 7a800d7016..4892feeea9 100644 --- a/snuba/cli/devserver.py +++ b/snuba/cli/devserver.py @@ -112,8 +112,8 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: [ "snuba", "rust-consumer", - "--storage=spans_v2", - "--consumer-group=spans_v2_group", + "--storage=eap_spans", + "--consumer-group=eap_spans_group", "--use-rust-processor", *COMMON_RUST_CONSUMER_DEV_OPTIONS, ], diff --git a/snuba/datasets/processors/spans_v2_processor.py b/snuba/datasets/processors/spans_v2_processor.py index 8babef7e85..8350bbb486 100644 --- a/snuba/datasets/processors/spans_v2_processor.py +++ b/snuba/datasets/processors/spans_v2_processor.py @@ -1,6 +1,6 @@ from snuba.datasets.processors.rust_compat_processor import RustCompatProcessor -class SpansV2MessageProcessor(RustCompatProcessor): +class EAPSpansMessageProcessor(RustCompatProcessor): def __init__(self) -> None: - super().__init__("SpansV2MessageProcessor") + super().__init__("EAPSpansMessageProcessor") diff --git a/tests/datasets/test_spans_processor.py b/tests/datasets/test_spans_processor.py index 8b4a4b054d..1e1e1da964 100644 --- a/tests/datasets/test_spans_processor.py +++ b/tests/datasets/test_spans_processor.py @@ -52,6 +52,7 @@ def serialize(self) -> SpanEvent: "is_segment": False, "parent_span_id": self.parent_span_id, "project_id": self.project_id, + "organization_id": 1, "received": self.received, "retention_days": self.retention_days, "segment_id": self.segment_id, From 77f7f3183aee5799e586bb57a569edf03e7b510f Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Mon, 22 Jul 2024 13:23:30 -0400 Subject: [PATCH 12/17] lint --- rust_snuba/src/processors/eap_spans.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust_snuba/src/processors/eap_spans.rs b/rust_snuba/src/processors/eap_spans.rs index d16e2780d2..470ad451ef 100644 --- a/rust_snuba/src/processors/eap_spans.rs +++ b/rust_snuba/src/processors/eap_spans.rs @@ -281,8 +281,6 @@ fn half_md5(input: &[u8]) -> u64 { } impl From for EAPSpan { - type Error = anyhow::Error; - fn from(from: FromSpanMessage) -> EAPSpan { let sentry_tags = from.sentry_tags.unwrap_or_default(); let tags = from.tags.unwrap_or_default(); From d514f4b348193e5c8bc16a2188058286bef212a4 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Mon, 22 Jul 2024 13:48:40 -0400 Subject: [PATCH 13/17] snapshot test --- ...EAPSpansMessageProcessor-snuba-spans__1__basic_span.json.snap} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename rust_snuba/src/processors/snapshots/{rust_snuba__processors__tests__schemas@snuba-spans-SpansV2MessageProcessor-snuba-spans__1__basic_span.json.snap => rust_snuba__processors__tests__schemas@snuba-spans-EAPSpansMessageProcessor-snuba-spans__1__basic_span.json.snap} (100%) diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-spans-SpansV2MessageProcessor-snuba-spans__1__basic_span.json.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-spans-EAPSpansMessageProcessor-snuba-spans__1__basic_span.json.snap similarity index 100% rename from rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-spans-SpansV2MessageProcessor-snuba-spans__1__basic_span.json.snap rename to rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-spans-EAPSpansMessageProcessor-snuba-spans__1__basic_span.json.snap From c3055f91fcc9f644f3ab67983be4b4e4db405ba8 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Mon, 22 Jul 2024 13:49:12 -0400 Subject: [PATCH 14/17] more eap --- snuba/cli/devserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snuba/cli/devserver.py b/snuba/cli/devserver.py index 4892feeea9..ac8fc73c74 100644 --- a/snuba/cli/devserver.py +++ b/snuba/cli/devserver.py @@ -108,7 +108,7 @@ def devserver(*, bootstrap: bool, workers: bool) -> None: ], ), ( - "spans-v2-consumer", + "eap-spans-consumer", [ "snuba", "rust-consumer", From 3bfb6790d7df47a575329156b3523902b1e77f8a Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Mon, 22 Jul 2024 14:04:38 -0400 Subject: [PATCH 15/17] MORE BUCKETS --- rust_snuba/src/processors/eap_spans.rs | 82 +++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/rust_snuba/src/processors/eap_spans.rs b/rust_snuba/src/processors/eap_spans.rs index 470ad451ef..c9f82178a8 100644 --- a/rust_snuba/src/processors/eap_spans.rs +++ b/rust_snuba/src/processors/eap_spans.rs @@ -273,11 +273,91 @@ struct EAPSpan { attr_bool_8: HashMap, #[serde(skip_serializing_if = "HashMap::is_empty")] attr_bool_9: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_10: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_11: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_12: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_13: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_14: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_15: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_16: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_17: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_18: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_19: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_20: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_21: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_22: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_23: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_24: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_25: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_26: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_27: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_28: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_29: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_30: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_31: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_32: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_33: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_34: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_35: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_36: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_37: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_38: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_39: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_40: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_41: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_42: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_43: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_44: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_45: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_46: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_47: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_48: HashMap, + #[serde(skip_serializing_if = "HashMap::is_empty")] + attr_bool_49: HashMap, } fn half_md5(input: &[u8]) -> u64 { let full_hash = md5::compute(input).0; - u64::from_be_bytes(full_hash[0..8].try_into().expect("should not happen")) + u64::from_be_bytes(full_hash[0..8].try_into().unwrap()) } impl From for EAPSpan { From 8e376e8e7371ea1dcb4d7e6e5d0680d9bd5d76d7 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Mon, 22 Jul 2024 14:09:21 -0400 Subject: [PATCH 16/17] eap test --- ... rust_snuba__processors__eap_spans__tests__serialization.snap} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename rust_snuba/src/processors/snapshots/{rust_snuba__processors__spans_v2__tests__serialization.snap => rust_snuba__processors__eap_spans__tests__serialization.snap} (100%) diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__eap_spans__tests__serialization.snap similarity index 100% rename from rust_snuba/src/processors/snapshots/rust_snuba__processors__spans_v2__tests__serialization.snap rename to rust_snuba/src/processors/snapshots/rust_snuba__processors__eap_spans__tests__serialization.snap From 142713ab76eb485abce5b392ea15198a160dbeb6 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Mon, 22 Jul 2024 14:35:49 -0400 Subject: [PATCH 17/17] Fix tests --- ...snuba__processors__tests__snuba-spans-.snap | 18 ++++++++++++++++++ rust_snuba/src/processors/spans.rs | 2 ++ 2 files changed, 20 insertions(+) create mode 100644 rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__snuba-spans-.snap diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__snuba-spans-.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__snuba-spans-.snap new file mode 100644 index 0000000000..5f85557705 --- /dev/null +++ b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__snuba-spans-.snap @@ -0,0 +1,18 @@ +--- +source: src/processors/mod.rs +expression: diff +--- +[ + Change { + path: "", + change: RequiredAdd { + property: "end_timestamp_precise", + }, + }, + Change { + path: "", + change: RequiredAdd { + property: "organization_id", + }, + }, +] diff --git a/rust_snuba/src/processors/spans.rs b/rust_snuba/src/processors/spans.rs index c8ceccec89..1f41fafbc8 100644 --- a/rust_snuba/src/processors/spans.rs +++ b/rust_snuba/src/processors/spans.rs @@ -344,6 +344,7 @@ mod tests { parent_span_id: Option, profile_id: Option, project_id: Option, + organization_id: Option, received: Option, retention_days: Option, segment_id: Option, @@ -366,6 +367,7 @@ mod tests { parent_span_id: Some("deadbeefdeadbeef".into()), profile_id: Some(Uuid::new_v4()), project_id: Some(1), + organization_id: Some(1), retention_days: Some(90), received: Some(1691105878.720), segment_id: Some("deadbeefdeadbeef".into()),