From d6514184e8d0b6654d73cb1787da5e5370d40272 Mon Sep 17 00:00:00 2001 From: Katie Byers Date: Mon, 23 Sep 2024 10:58:26 -0700 Subject: [PATCH 1/8] add seer fields to grouphash metadata table --- migrations_lockfile.txt | 2 +- ...5_add_seer_fields_to_grouphash_metadata.py | 70 +++++++++++++++++++ src/sentry/models/grouphashmetadata.py | 36 ++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 src/sentry/migrations/0765_add_seer_fields_to_grouphash_metadata.py diff --git a/migrations_lockfile.txt b/migrations_lockfile.txt index 5b94065e474a3..66d72b8ff4fed 100644 --- a/migrations_lockfile.txt +++ b/migrations_lockfile.txt @@ -10,7 +10,7 @@ hybridcloud: 0016_add_control_cacheversion nodestore: 0002_nodestore_no_dictfield remote_subscriptions: 0003_drop_remote_subscription replays: 0004_index_together -sentry: 0764_migrate_bad_status_substatus_rows +sentry: 0765_add_seer_fields_to_grouphash_metadata social_auth: 0002_default_auto_field uptime: 0013_uptime_subscription_new_unique workflow_engine: 0005_data_source_detector diff --git a/src/sentry/migrations/0765_add_seer_fields_to_grouphash_metadata.py b/src/sentry/migrations/0765_add_seer_fields_to_grouphash_metadata.py new file mode 100644 index 0000000000000..5a4eb3f17582b --- /dev/null +++ b/src/sentry/migrations/0765_add_seer_fields_to_grouphash_metadata.py @@ -0,0 +1,70 @@ +# Generated by Django 5.1.1 on 2024-09-23 15:40 + +import django.db.models.deletion +from django.db import migrations, models + +import sentry.db.models.fields.foreignkey +from sentry.new_migrations.migrations import CheckedMigration + + +class Migration(CheckedMigration): + # This flag is used to mark that a migration shouldn't be automatically run in production. + # This should only be used for operations where it's safe to run the migration after your + # code has deployed. So this should not be used for most operations that alter the schema + # of a table. + # Here are some things that make sense to mark as post deployment: + # - Large data migrations. Typically we want these to be run manually so that they can be + # monitored and not block the deploy for a long period of time while they run. + # - Adding indexes to large tables. Since this can take a long time, we'd generally prefer to + # run this outside deployments so that we don't block them. Note that while adding an index + # is a schema change, it's completely safe to run the operation after the code has deployed. + # Once deployed, run these manually via: https://develop.sentry.dev/database-migrations/#migration-deployment + + is_post_deployment = False + + dependencies = [ + ("sentry", "0764_migrate_bad_status_substatus_rows"), + ] + + operations = [ + migrations.AddField( + model_name="grouphashmetadata", + name="seer_date_sent", + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name="grouphashmetadata", + name="seer_event_sent", + field=models.CharField(max_length=32, null=True), + ), + migrations.AddField( + model_name="grouphashmetadata", + name="seer_grouphash_sent", + field=sentry.db.models.fields.foreignkey.FlexibleForeignKey( + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="+", + to="sentry.grouphash", + ), + ), + migrations.AddField( + model_name="grouphashmetadata", + name="seer_match_distance", + field=models.FloatField(null=True), + ), + migrations.AddField( + model_name="grouphashmetadata", + name="seer_matched_grouphash", + field=sentry.db.models.fields.foreignkey.FlexibleForeignKey( + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="seer_matchees", + to="sentry.grouphash", + ), + ), + migrations.AddField( + model_name="grouphashmetadata", + name="seer_model", + field=models.CharField(null=True), + ), + ] diff --git a/src/sentry/models/grouphashmetadata.py b/src/sentry/models/grouphashmetadata.py index b661e2178c0c2..52a6a5a88042d 100644 --- a/src/sentry/models/grouphashmetadata.py +++ b/src/sentry/models/grouphashmetadata.py @@ -4,6 +4,7 @@ from sentry.backup.scopes import RelocationScope from sentry.db.models import Model, region_silo_model from sentry.db.models.base import sane_repr +from sentry.db.models.fields.foreignkey import FlexibleForeignKey @region_silo_model @@ -11,11 +12,46 @@ class GroupHashMetadata(Model): __relocation_scope__ = RelocationScope.Excluded # GENERAL + grouphash = models.OneToOneField( "sentry.GroupHash", related_name="_metadata", on_delete=models.CASCADE ) date_added = models.DateTimeField(default=timezone.now) + # SEER + + # Only one hash representing each group is sent to Seer. For the grouphash actually sent, this + # field and the `grouphash` field will be identical. For the grouphashes assigned to the same + # group but which aren't sent, this will point to the GroupHash record for the sent hash. Note + # that because of merging/unmerging, the sent GroupHash and this metadata's GroupHash (if not + # one and the same) aren't guaranteed to forever point to the same group (though they will when + # this field is written). + seer_grouphash_sent = FlexibleForeignKey( + "sentry.GroupHash", + # If we end up needing to reference in this direction, we can handle it with a property on + # GroupHash + related_name="+", + on_delete=models.DO_NOTHING, + null=True, + ) + + # NOTE: The rest of the Seer-related fields are only stored on the metadata of the GroupHash + # actually sent to Seer. + + # When this hash was sent to Seer. This will be different than `date_added` if we send it to + # Seer as part of a backfill rather than during ingest. + seer_date_sent = models.DateTimeField(null=True) + # Id of the event whose stacktrace was sent to Seer + seer_event_sent = models.CharField(max_length=32, null=True) + # The version of the Seer model used to process this hash value + seer_model = models.CharField(null=True) + # The `GroupHash` record representing the match Seer sent back as a match (if any) + seer_matched_grouphash = FlexibleForeignKey( + "sentry.GroupHash", related_name="seer_matchees", on_delete=models.DO_NOTHING, null=True + ) + # The similarity between this hash's stacktrace and the parent (matched) hash's stacktrace + seer_match_distance = models.FloatField(null=True) + class Meta: app_label = "sentry" db_table = "sentry_grouphashmetadata" From 51ac901eb2bd72a71cbb642c037481b1d349f550 Mon Sep 17 00:00:00 2001 From: Katie Byers Date: Mon, 23 Sep 2024 10:58:31 -0700 Subject: [PATCH 2/8] pass `all_grouphashes` to `maybe_check_seer_for_matching_grouphash` --- src/sentry/event_manager.py | 4 ++-- src/sentry/grouping/ingest/seer.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/sentry/event_manager.py b/src/sentry/event_manager.py index 8158b9c23dc25..3c3aa5b99d30e 100644 --- a/src/sentry/event_manager.py +++ b/src/sentry/event_manager.py @@ -1445,7 +1445,7 @@ def _save_aggregate( # If we still haven't found a matching grouphash, we're now safe to go ahead and talk to # seer and/or create the group. if existing_grouphash is None: - seer_matched_grouphash = maybe_check_seer_for_matching_grouphash(event) + seer_matched_grouphash = maybe_check_seer_for_matching_grouphash(event, grouphashes) seer_matched_group = ( Group.objects.filter(id=seer_matched_grouphash.group_id).first() if seer_matched_grouphash @@ -1614,7 +1614,7 @@ def _save_aggregate_new( result = "found_secondary" # If we still haven't found a group, ask Seer for a match (if enabled for the project) else: - seer_matched_grouphash = maybe_check_seer_for_matching_grouphash(event) + seer_matched_grouphash = maybe_check_seer_for_matching_grouphash(event, all_grouphashes) if seer_matched_grouphash: group_info = handle_existing_grouphash(job, seer_matched_grouphash, all_grouphashes) diff --git a/src/sentry/grouping/ingest/seer.py b/src/sentry/grouping/ingest/seer.py index 34a6aa9552a59..d3f3c5cf6981a 100644 --- a/src/sentry/grouping/ingest/seer.py +++ b/src/sentry/grouping/ingest/seer.py @@ -232,7 +232,9 @@ def get_seer_similar_issues( return (similar_issues_metadata, parent_grouphash) -def maybe_check_seer_for_matching_grouphash(event: Event) -> GroupHash | None: +def maybe_check_seer_for_matching_grouphash( + event: Event, all_grouphashes: list[GroupHash] +) -> GroupHash | None: seer_matched_grouphash = None if should_call_seer_for_grouping(event): From f14e593e0ff89700442bb17693c6ad820ee6b5c8 Mon Sep 17 00:00:00 2001 From: Katie Byers Date: Mon, 23 Sep 2024 10:58:35 -0700 Subject: [PATCH 3/8] stop storing seer results in event data --- src/sentry/grouping/ingest/seer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sentry/grouping/ingest/seer.py b/src/sentry/grouping/ingest/seer.py index d3f3c5cf6981a..bc520927c646d 100644 --- a/src/sentry/grouping/ingest/seer.py +++ b/src/sentry/grouping/ingest/seer.py @@ -247,7 +247,6 @@ def maybe_check_seer_for_matching_grouphash( # If no matching group is found in Seer, we'll still get back result # metadata, but `seer_matched_grouphash` will be None seer_response_data, seer_matched_grouphash = get_seer_similar_issues(event) - event.data["seer_similarity"] = seer_response_data # Insurance - in theory we shouldn't ever land here except Exception as e: From 4fdbdf1df27417aae73dad913b8d8f890511a79f Mon Sep 17 00:00:00 2001 From: Katie Byers Date: Mon, 23 Sep 2024 10:58:37 -0700 Subject: [PATCH 4/8] remove logs during group creation --- src/sentry/event_manager.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/sentry/event_manager.py b/src/sentry/event_manager.py index 3c3aa5b99d30e..ca749320db603 100644 --- a/src/sentry/event_manager.py +++ b/src/sentry/event_manager.py @@ -1806,16 +1806,6 @@ def _create_group( first_release: Release | None = None, **group_creation_kwargs: Any, ) -> Group: - # Temporary log to debug events seeming to disappear after being sent to Seer - if event.data.get("seer_similarity"): - logger.info( - "seer.similarity.pre_create_group", - extra={ - "event_id": event.event_id, - "hash": event.get_primary_hash(), - "project": project.id, - }, - ) short_id = _get_next_short_id(project) @@ -1891,18 +1881,6 @@ def _create_group( logger.exception("Error after unsticking project counter") raise - # Temporary log to debug events seeming to disappear after being sent to Seer - if event.data.get("seer_similarity"): - logger.info( - "seer.similarity.post_create_group", - extra={ - "event_id": event.event_id, - "hash": event.get_primary_hash(), - "project": project.id, - "group_id": group.id, - }, - ) - return group From c6ae62d14c0a44a02a3a743367ee486db6a89ea4 Mon Sep 17 00:00:00 2001 From: Katie Byers Date: Mon, 23 Sep 2024 10:58:39 -0700 Subject: [PATCH 5/8] store seer data in grouphash metadata --- src/sentry/grouping/ingest/seer.py | 39 +++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/src/sentry/grouping/ingest/seer.py b/src/sentry/grouping/ingest/seer.py index bc520927c646d..64df1bbcbc4c3 100644 --- a/src/sentry/grouping/ingest/seer.py +++ b/src/sentry/grouping/ingest/seer.py @@ -243,15 +243,48 @@ def maybe_check_seer_for_matching_grouphash( sample_rate=options.get("seer.similarity.metrics_sample_rate"), tags={"call_made": True, "blocker": "none"}, ) + try: # If no matching group is found in Seer, we'll still get back result # metadata, but `seer_matched_grouphash` will be None seer_response_data, seer_matched_grouphash = get_seer_similar_issues(event) - - # Insurance - in theory we shouldn't ever land here - except Exception as e: + except Exception as e: # Insurance - in theory we shouldn't ever land here sentry_sdk.capture_exception( e, tags={"event": event.event_id, "project": event.project.id} ) + return None + + # Find the GroupHash for the hash value sent to Seer + primary_hash = event.get_primary_hash() + grouphash_sent = list( + filter(lambda grouphash: grouphash.hash == primary_hash, all_grouphashes) + )[0] + + # Update GroupHashes with Seer results + for grouphash in all_grouphashes: + metadata = grouphash.metadata + + if metadata: + # Mark all the GroupHashes as having been represented by the one we sent + metadata.seer_grouphash_sent = grouphash_sent + + # Store the Seer results only on the GroupHash which was actually sent + if grouphash is grouphash_sent: + # Technically the time of the metadata record creation and the time of the Seer + # request will be some milliseconds apart, but the difference isn't meaningful + # and forcing them to be the same (rather than just close) lets us use their + # equality as a signal that the Seer call happened during ingest rather than + # during a backfill, without having to store that information separately + metadata.seer_date_sent = metadata.date_added + metadata.seer_event_sent = event.event_id + metadata.seer_model = seer_response_data["similarity_model_version"] + metadata.seer_matched_grouphash = seer_matched_grouphash + metadata.seer_match_distance = ( + seer_response_data["results"][0]["stacktrace_distance"] + if seer_matched_grouphash + else None + ) + + metadata.save() return seer_matched_grouphash From 1e52934994901a719ae5cb650ee8c43c1ff17425 Mon Sep 17 00:00:00 2001 From: Katie Byers Date: Mon, 23 Sep 2024 10:58:42 -0700 Subject: [PATCH 6/8] fix tests for seer metadata storage --- .../grouping/test_seer_grouping.py | 177 ++++++++++++++---- 1 file changed, 145 insertions(+), 32 deletions(-) diff --git a/tests/sentry/event_manager/grouping/test_seer_grouping.py b/tests/sentry/event_manager/grouping/test_seer_grouping.py index 6aef13fa8e83e..936a5db654b05 100644 --- a/tests/sentry/event_manager/grouping/test_seer_grouping.py +++ b/tests/sentry/event_manager/grouping/test_seer_grouping.py @@ -6,9 +6,11 @@ from sentry.conf.server import SEER_SIMILARITY_MODEL_VERSION from sentry.grouping.ingest.seer import get_seer_similar_issues, should_call_seer_for_grouping from sentry.models.grouphash import GroupHash +from sentry.projectoptions.defaults import DEFAULT_GROUPING_CONFIG, LEGACY_GROUPING_CONFIG from sentry.seer.similarity.types import SeerSimilarIssueData from sentry.testutils.cases import TestCase from sentry.testutils.helpers.eventprocessing import save_new_event +from sentry.testutils.helpers.features import with_feature from sentry.testutils.pytest.mocking import capture_results from sentry.utils.types import NonNone @@ -57,9 +59,7 @@ def test_obeys_seer_similarity_flags(self): assert should_call_seer_spy.call_count == 1 assert get_seer_similar_issues_spy.call_count == 0 - # No metadata stored, parent group not used (even though `should_group` is True) - assert "seer_similarity" not in NonNone(new_event.group).data["metadata"] - assert "seer_similarity" not in new_event.data + # Parent group not used (even though `should_group` is True) assert new_event.group_id != existing_event.group_id should_call_seer_spy.reset_mock() @@ -83,9 +83,9 @@ def test_obeys_seer_similarity_flags(self): assert should_call_seer_spy.call_count == 1 assert get_seer_similar_issues_spy.call_count == 1 - # Metadata returned and stored + # Metadata returned (metadata storage is tested separately in + # `test_stores_seer_results_in_grouphash_metadata`) assert get_seer_similar_issues_return_values[0][0] == expected_metadata - assert new_event.data["seer_similarity"] == expected_metadata # Parent grouphash returned and parent group used assert get_seer_similar_issues_return_values[0][1] == expected_grouphash @@ -128,37 +128,150 @@ def test_bypasses_seer_if_group_found(self, mock_get_seer_similar_issues: MagicM mock_get_seer_similar_issues.reset_mock() + @with_feature("organizations:grouphash-metadata-creation") @patch("sentry.grouping.ingest.seer.should_call_seer_for_grouping", return_value=True) - def test_stores_seer_results_in_metadata(self, _): - for use_optimized_grouping, existing_event_message, new_event_message in [ - (True, "Dogs are great!", "Adopt don't shop"), - (False, "Maisey is silly", "Charlie is goofy"), - ]: - with patch( - "sentry.event_manager.project_uses_optimized_grouping", - return_value=use_optimized_grouping, - ): - existing_event = save_new_event({"message": existing_event_message}, self.project) + def test_stores_seer_results_in_grouphash_metadata(self, _): + def assert_correct_seer_metadata( + grouphash, + expected_seer_grouphash_sent, + expected_seer_date_sent, + expected_seer_event_sent, + expected_seer_model, + expected_seer_matched_grouphash, + expected_seer_match_distance, + ): + metadata = grouphash.metadata - seer_result_data = SeerSimilarIssueData( - parent_hash=NonNone(existing_event.get_primary_hash()), - parent_group_id=NonNone(existing_event.group_id), - stacktrace_distance=0.01, - message_distance=0.05, - should_group=True, - ) + assert metadata + assert metadata.seer_grouphash_sent == expected_seer_grouphash_sent + assert metadata.seer_date_sent == expected_seer_date_sent + assert metadata.seer_event_sent == expected_seer_event_sent + assert metadata.seer_model == expected_seer_model + assert metadata.seer_matched_grouphash == expected_seer_matched_grouphash + assert metadata.seer_match_distance == expected_seer_match_distance - with patch( - "sentry.grouping.ingest.seer.get_similarity_data_from_seer", - return_value=[seer_result_data], - ): - new_event = save_new_event({"message": new_event_message}, self.project) - expected_metadata = { - "similarity_model_version": SEER_SIMILARITY_MODEL_VERSION, - "results": [asdict(seer_result_data)], - } + # The overall idea(s) of this test: + # + # - Send three events: an event which gets sent to Seer but doesn't find a match, an event + # which gets sent and does find a match, and an event which doesn't get sent at all, to + # test what Seer metadata gets stored in each case. + # + # - Set the project to be in a grouping config transition so that primary and secondary + # hashes will both be calculated, and include numbers in the message of one of the events + # sent to Seer so that the primary and secondary hashes will be different (since the + # legacy config won't parameterize the numbers). This will let us test the additional + # case of a GroupHash which isn't sent to Seer but whose sibling GroupHash is sent. + + self.project.update_option("sentry:grouping_config", DEFAULT_GROUPING_CONFIG) + self.project.update_option("sentry:secondary_grouping_config", LEGACY_GROUPING_CONFIG) + self.project.update_option("sentry:secondary_grouping_expiry", time() + 3600) + + # First, the event which is sent but finds no match: + + with patch( + "sentry.grouping.ingest.seer.get_similarity_data_from_seer", + return_value=[], + ) as mock_get_similarity_data_from_seer: + existing_event = save_new_event({"message": "Dogs are great! 11211231"}, self.project) + + existing_event_grouphashes = GroupHash.objects.filter( + group_id=NonNone(existing_event.group_id) + ) + assert len(existing_event_grouphashes) == 2 + + existing_event_primary_grouphash = existing_event_grouphashes.filter( + hash=existing_event.get_primary_hash() + ).first() + assert existing_event_primary_grouphash + assert existing_event_primary_grouphash.metadata + # Since `existing_event_grouphashes` only has two members, excluding the primary + # necessarily gives us the secondary + existing_event_secondary_grouphash = existing_event_grouphashes.exclude( + hash=existing_event.get_primary_hash() + ).first() + assert existing_event_secondary_grouphash + assert existing_event_secondary_grouphash.metadata + + # Make sure we're right about which grouphash was sent + assert ( + mock_get_similarity_data_from_seer.call_args.args[0]["hash"] + == existing_event_primary_grouphash.hash + ) + + # GroupHash sent to Seer + assert_correct_seer_metadata( + existing_event_primary_grouphash, + existing_event_primary_grouphash, + existing_event_primary_grouphash.metadata.date_added, + existing_event.event_id, + SEER_SIMILARITY_MODEL_VERSION, + None, + None, + ) + # Sibling GroupHash + assert_correct_seer_metadata( + existing_event_secondary_grouphash, + existing_event_primary_grouphash, + None, + None, + None, + None, + None, + ) + + # Next, the event which is sent and does find a match: + + seer_result_data = SeerSimilarIssueData( + parent_hash=NonNone(existing_event.get_primary_hash()), + parent_group_id=NonNone(existing_event.group_id), + stacktrace_distance=0.01, + message_distance=0.05, + should_group=True, + ) + + with patch( + "sentry.grouping.ingest.seer.get_similarity_data_from_seer", + return_value=[seer_result_data], + ): + new_event = save_new_event({"message": "Adopt don't shop"}, self.project) + + assert new_event.group_id == existing_event.group_id + + # Only one GroupHash was added this time + new_event_grouphashes = GroupHash.objects.filter( + group_id=NonNone(existing_event.group_id) + ).exclude(hash__in=[gh.hash for gh in existing_event_grouphashes]) + assert len(new_event_grouphashes) == 1 + + new_event_grouphash = new_event_grouphashes.first() + assert new_event_grouphash + assert new_event_grouphash.metadata + + # Just to confirm it's the right one + assert new_event_grouphash.hash == new_event.get_primary_hash() + + assert_correct_seer_metadata( + new_event_grouphash, + new_event_grouphash, + new_event_grouphash.metadata.date_added, + new_event.event_id, + SEER_SIMILARITY_MODEL_VERSION, + existing_event_primary_grouphash, + seer_result_data.stacktrace_distance, + ) + + # Finally, the event which isn't sent at all: + + with patch("sentry.grouping.ingest.seer.should_call_seer_for_grouping", return_value=False): + third_event = save_new_event({"message": "Sit! Stay! Good dog!"}, self.project) + + third_event_grouphash = GroupHash.objects.filter( + hash=third_event.get_primary_hash() + ).first() + assert third_event_grouphash + assert third_event_grouphash.metadata - assert new_event.data["seer_similarity"] == expected_metadata + assert_correct_seer_metadata(third_event_grouphash, None, None, None, None, None, None) @patch("sentry.grouping.ingest.seer.should_call_seer_for_grouping", return_value=True) def test_assigns_event_to_neighbor_group_if_found(self, _): From bd7682c957301246b4004266da846c0ce7de562e Mon Sep 17 00:00:00 2001 From: Katie Byers Date: Mon, 23 Sep 2024 10:58:45 -0700 Subject: [PATCH 7/8] fix model dependency fixtures --- fixtures/backup/model_dependencies/detailed.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fixtures/backup/model_dependencies/detailed.json b/fixtures/backup/model_dependencies/detailed.json index 1e5510745364f..dc65ff93045c6 100644 --- a/fixtures/backup/model_dependencies/detailed.json +++ b/fixtures/backup/model_dependencies/detailed.json @@ -2319,6 +2319,16 @@ "kind": "DefaultOneToOneField", "model": "sentry.grouphash", "nullable": false + }, + "seer_grouphash_sent": { + "kind": "FlexibleForeignKey", + "model": "sentry.grouphash", + "nullable": true + }, + "seer_matched_grouphash": { + "kind": "FlexibleForeignKey", + "model": "sentry.grouphash", + "nullable": true } }, "model": "sentry.grouphashmetadata", From 32565d6560fab94de7197717a7eb7aafd681cb05 Mon Sep 17 00:00:00 2001 From: Katie Byers Date: Mon, 23 Sep 2024 10:58:48 -0700 Subject: [PATCH 8/8] update comparator snapshot --- .../test_comparators/test_default_comparators.pysnap | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/sentry/backup/snapshots/test_comparators/test_default_comparators.pysnap b/tests/sentry/backup/snapshots/test_comparators/test_default_comparators.pysnap index c204e6ae97630..e6809c3c9d27b 100644 --- a/tests/sentry/backup/snapshots/test_comparators/test_default_comparators.pysnap +++ b/tests/sentry/backup/snapshots/test_comparators/test_default_comparators.pysnap @@ -1,5 +1,5 @@ --- -created: '2024-09-17T12:19:51.014511+00:00' +created: '2024-09-23T17:48:55.681597+00:00' creator: sentry source: tests/sentry/backup/test_comparators.py --- @@ -579,6 +579,8 @@ source: tests/sentry/backup/test_comparators.py - class: ForeignKeyComparator fields: - grouphash + - seer_grouphash_sent + - seer_matched_grouphash model_name: sentry.grouphashmetadata - comparators: - class: ForeignKeyComparator