From fa2fcd96b2b4d1135420b7827877f30a64f501a0 Mon Sep 17 00:00:00 2001 From: Snigdha Sharma Date: Thu, 29 Aug 2024 14:49:25 -0700 Subject: [PATCH] chore(issues): Backfill IGNORED groups with missing substatuses (#75702) We need to run a backfill to fix the missing substatuses for IGNORED groups. This [redash query](https://redash.getsentry.net/queries/6898) shows we have ~120K ignored groups that have no substatus. Further digging shows that we _should_ have the info needed to backfill via Activity and GroupSnooze tables. Marking this as a post-deploy migration for safety. Fixes https://github.com/getsentry/sentry/issues/75684 --- migrations_lockfile.txt | 2 +- .../0753_fix_substatus_for_ignored_groups.py | 107 ++++++++++++++++++ ...t_0753_fix_substatus_for_ignored_groups.py | 102 +++++++++++++++++ 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 src/sentry/migrations/0753_fix_substatus_for_ignored_groups.py create mode 100644 tests/sentry/migrations/test_0753_fix_substatus_for_ignored_groups.py diff --git a/migrations_lockfile.txt b/migrations_lockfile.txt index cffb04e9281e89..8b9947547222fc 100644 --- a/migrations_lockfile.txt +++ b/migrations_lockfile.txt @@ -10,6 +10,6 @@ hybridcloud: 0016_add_control_cacheversion nodestore: 0002_nodestore_no_dictfield remote_subscriptions: 0003_drop_remote_subscription replays: 0004_index_together -sentry: 0752_fix_substatus_for_unresolved_groups +sentry: 0753_fix_substatus_for_ignored_groups social_auth: 0002_default_auto_field uptime: 0007_update_detected_subscription_interval diff --git a/src/sentry/migrations/0753_fix_substatus_for_ignored_groups.py b/src/sentry/migrations/0753_fix_substatus_for_ignored_groups.py new file mode 100644 index 00000000000000..30c0f88abe0f24 --- /dev/null +++ b/src/sentry/migrations/0753_fix_substatus_for_ignored_groups.py @@ -0,0 +1,107 @@ +# Generated by Django 5.0.7 on 2024-08-05 17:50 + +from django.apps.registry import Apps +from django.db import migrations +from django.db.backends.base.schema import BaseDatabaseSchemaEditor + +from sentry.new_migrations.migrations import CheckedMigration +from sentry.utils.query import RangeQuerySetWrapperWithProgressBarApprox + +# Copying constants defined in the models + + +class ActivityType: + SET_IGNORED = 3 + + +class GroupHistoryStatus: + ARCHIVED_UNTIL_ESCALATING = 15 + ARCHIVED_FOREVER = 16 + ARCHIVED_UNTIL_CONDITION_MET = 17 + + +class GroupSubStatus: + UNTIL_ESCALATING = 1 + # Group is ignored/archived for a count/user count/duration + UNTIL_CONDITION_MET = 4 + # Group is ignored/archived forever + FOREVER = 5 + + +class GroupStatus: + IGNORED = 2 + + +# End copy + +ACTIVITY_DATA_FIELDS = { + "ignoreCount", + "ignoreDuration", + "ignoreUntil", + "ignoreUserCount", + "ignoreUserWindow", + "ignoreWindow", +} + + +def backfill_substatus_for_ignored_groups( + apps: Apps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + Group = apps.get_model("sentry", "Group") + Activity = apps.get_model("sentry", "Activity") + GroupSnooze = apps.get_model("sentry", "GroupSnooze") + + activity = Activity.objects.filter(type=ActivityType.SET_IGNORED) + for group in RangeQuerySetWrapperWithProgressBarApprox( + Group.objects.filter(status=GroupStatus.IGNORED, substatus=None), + ): + group_activity = activity.filter(group_id=group.id).order_by("-datetime").first() + new_substatus = None + if group_activity: + # If ignoreUntilEscalating is set, we should set the substatus to UNTIL_ESCALATING + if group_activity.data.get("ignoreUntilEscalating", False): + new_substatus = GroupSubStatus.UNTIL_ESCALATING + # If any other field in the activity data is set, we should set the substatus to UNTIL_CONDITION_MET + elif any(group_activity.data.get(field) for field in ACTIVITY_DATA_FIELDS): + new_substatus = GroupSubStatus.UNTIL_CONDITION_MET + + # If no activity is found or the activity data is not set, check the group snooze table + if not new_substatus: + snooze = GroupSnooze.objects.filter(group=group) + if snooze.exists(): + # If snooze exists, we should set the substatus to UNTIL_CONDITION_MET + new_substatus = GroupSubStatus.UNTIL_CONDITION_MET + else: + # If we have no other information stored about the group's status conditions, the group is ignored forever + new_substatus = GroupSubStatus.FOREVER + + group.substatus = new_substatus + group.save(update_fields=["substatus"]) + + +class Migration(CheckedMigration): + # This flag is used to mark that a migration shouldn't be automatically run in production. + # This should only be used for operations where it's safe to run the migration after your + # code has deployed. So this should not be used for most operations that alter the schema + # of a table. + # Here are some things that make sense to mark as post deployment: + # - Large data migrations. Typically we want these to be run manually so that they can be + # monitored and not block the deploy for a long period of time while they run. + # - Adding indexes to large tables. Since this can take a long time, we'd generally prefer to + # run this outside deployments so that we don't block them. Note that while adding an index + # is a schema change, it's completely safe to run the operation after the code has deployed. + # Once deployed, run these manually via: https://develop.sentry.dev/database-migrations/#migration-deployment + + is_post_deployment = True + + dependencies = [ + ("sentry", "0752_fix_substatus_for_unresolved_groups"), + ] + + operations = [ + migrations.RunPython( + backfill_substatus_for_ignored_groups, + migrations.RunPython.noop, + hints={"tables": ["sentry_groupedmessage"]}, + ), + ] diff --git a/tests/sentry/migrations/test_0753_fix_substatus_for_ignored_groups.py b/tests/sentry/migrations/test_0753_fix_substatus_for_ignored_groups.py new file mode 100644 index 00000000000000..d0e0ace5764b48 --- /dev/null +++ b/tests/sentry/migrations/test_0753_fix_substatus_for_ignored_groups.py @@ -0,0 +1,102 @@ +from sentry.models.activity import Activity +from sentry.models.group import Group, GroupStatus +from sentry.models.groupsnooze import GroupSnooze +from sentry.models.organization import Organization +from sentry.testutils.cases import TestMigrations +from sentry.types.activity import ActivityType +from sentry.types.group import GroupSubStatus + + +class FixSubstatusForIgnoreedGroupsTest(TestMigrations): + migrate_from = "0752_fix_substatus_for_unresolved_groups" + migrate_to = "0753_fix_substatus_for_ignored_groups" + + def setup_before_migration(self, app): + self.organization = Organization.objects.create(name="test", slug="test") + self.project = self.create_project(organization=self.organization) + self.do_not_update = Group.objects.create( + project=self.project, + status=GroupStatus.IGNORED, + substatus=GroupSubStatus.UNTIL_ESCALATING, + ) + + self.ignored_until_condition_met = Group.objects.create( + project=self.project, + status=GroupStatus.IGNORED, + ) + # .update() skips calling the pre_save checks which requires a substatus + self.ignored_until_condition_met.update(substatus=None) + self.ignored_until_condition_met.refresh_from_db() + assert self.ignored_until_condition_met.substatus is None + Activity.objects.create( + group=self.ignored_until_condition_met, + project=self.project, + type=ActivityType.SET_IGNORED.value, + data={"ignoreCount": 10}, + ) + + self.ignored_until_condition_met_no_activity = Group.objects.create( + project=self.project, + status=GroupStatus.IGNORED, + ) + self.ignored_until_condition_met_no_activity.update(substatus=None) + assert self.ignored_until_condition_met_no_activity.substatus is None + Activity.objects.create( + group=self.ignored_until_condition_met_no_activity, + project=self.project, + type=ActivityType.SET_IGNORED.value, + data={ + "ignoreCount": None, + "ignoreDuration": None, + "ignoreUntil": None, + "ignoreUserCount": None, + "ignoreUserWindow": None, + "ignoreWindow": None, + "ignoreUntilEscalating": None, + }, + ) + GroupSnooze.objects.create( + group=self.ignored_until_condition_met_no_activity, + count=10, + ) + + self.ignored_until_escalating = Group.objects.create( + project=self.project, + status=GroupStatus.IGNORED, + ) + # .update() skips calling the pre_save checks which requires a substatus + self.ignored_until_escalating.update(substatus=None) + self.ignored_until_escalating.refresh_from_db() + assert self.ignored_until_escalating.substatus is None + Activity.objects.create( + group=self.ignored_until_escalating, + project=self.project, + type=ActivityType.SET_IGNORED.value, + data={"ignoreUntilEscalating": True}, + ) + + self.ignored_forever = Group.objects.create( + project=self.project, + status=GroupStatus.IGNORED, + ) + self.ignored_forever.update(substatus=None) + assert self.ignored_forever.substatus is None + + def test(self): + self.do_not_update.refresh_from_db() + assert self.do_not_update.substatus == GroupSubStatus.UNTIL_ESCALATING + + self.ignored_until_condition_met.refresh_from_db() + assert self.ignored_until_condition_met.substatus == GroupSubStatus.UNTIL_CONDITION_MET + + self.ignored_until_condition_met_no_activity.refresh_from_db() + assert ( + self.ignored_until_condition_met_no_activity.substatus + == GroupSubStatus.UNTIL_CONDITION_MET + ) + + self.ignored_until_escalating.refresh_from_db() + assert self.ignored_until_escalating.substatus == GroupSubStatus.UNTIL_ESCALATING + + self.ignored_forever.refresh_from_db() + assert self.ignored_forever.substatus == GroupSubStatus.FOREVER