chore(issues): Backfill IGNORED groups with missing substatuses (#75702)

We need to run a backfill to fix the missing substatuses for IGNORED
groups. This [redash query](https://redash.getsentry.net/queries/6898)
shows we have ~120K ignored groups that have no substatus. Further
digging shows that we _should_ have the info needed to backfill via
Activity and GroupSnooze tables.

Marking this as a post-deploy migration for safety.

Fixes #75684

migrations_lockfile.txt

@@ -10,6 +10,6 @@ hybridcloud: 0016_add_control_cacheversion
 nodestore: 0002_nodestore_no_dictfield
 remote_subscriptions: 0003_drop_remote_subscription
 replays: 0004_index_together
-sentry: 0752_fix_substatus_for_unresolved_groups
+sentry: 0753_fix_substatus_for_ignored_groups
 social_auth: 0002_default_auto_field
 uptime: 0007_update_detected_subscription_interval

src/sentry/migrations/0753_fix_substatus_for_ignored_groups.py

@@ -0,0 +1,107 @@
+# Generated by Django 5.0.7 on 2024-08-05 17:50
+
+from django.apps.registry import Apps
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+
+from sentry.new_migrations.migrations import CheckedMigration
+from sentry.utils.query import RangeQuerySetWrapperWithProgressBarApprox
+
+# Copying constants defined in the models
+
+
+class ActivityType:
+ SET_IGNORED = 3
+
+
+class GroupHistoryStatus:
+ ARCHIVED_UNTIL_ESCALATING = 15
+ ARCHIVED_FOREVER = 16
+ ARCHIVED_UNTIL_CONDITION_MET = 17
+
+
+class GroupSubStatus:
+ UNTIL_ESCALATING = 1
+ # Group is ignored/archived for a count/user count/duration
+ UNTIL_CONDITION_MET = 4
+ # Group is ignored/archived forever
+ FOREVER = 5
+
+
+class GroupStatus:
+ IGNORED = 2
+
+
+# End copy
+
+ACTIVITY_DATA_FIELDS = {
+ "ignoreCount",
+ "ignoreDuration",
+ "ignoreUntil",
+ "ignoreUserCount",
+ "ignoreUserWindow",
+ "ignoreWindow",
+}
+
+
+def backfill_substatus_for_ignored_groups(
+ apps: Apps, schema_editor: BaseDatabaseSchemaEditor
+) -> None:
+ Group = apps.get_model("sentry", "Group")
+ Activity = apps.get_model("sentry", "Activity")
+ GroupSnooze = apps.get_model("sentry", "GroupSnooze")
+
+ activity = Activity.objects.filter(type=ActivityType.SET_IGNORED)
+ for group in RangeQuerySetWrapperWithProgressBarApprox(
+ Group.objects.filter(status=GroupStatus.IGNORED, substatus=None),
+ ):
+ group_activity = activity.filter(group_id=group.id).order_by("-datetime").first()
+ new_substatus = None
+ if group_activity:
+ # If ignoreUntilEscalating is set, we should set the substatus to UNTIL_ESCALATING
+ if group_activity.data.get("ignoreUntilEscalating", False):
+ new_substatus = GroupSubStatus.UNTIL_ESCALATING
+ # If any other field in the activity data is set, we should set the substatus to UNTIL_CONDITION_MET
+ elif any(group_activity.data.get(field) for field in ACTIVITY_DATA_FIELDS):
+ new_substatus = GroupSubStatus.UNTIL_CONDITION_MET
+
+ # If no activity is found or the activity data is not set, check the group snooze table
+ if not new_substatus:
+ snooze = GroupSnooze.objects.filter(group=group)
+ if snooze.exists():
+ # If snooze exists, we should set the substatus to UNTIL_CONDITION_MET
+ new_substatus = GroupSubStatus.UNTIL_CONDITION_MET
+ else:
+ # If we have no other information stored about the group's status conditions, the group is ignored forever
+ new_substatus = GroupSubStatus.FOREVER
+
+ group.substatus = new_substatus
+ group.save(update_fields=["substatus"])
+
+
+class Migration(CheckedMigration):
+ # This flag is used to mark that a migration shouldn't be automatically run in production.
+ # This should only be used for operations where it's safe to run the migration after your
+ # code has deployed. So this should not be used for most operations that alter the schema
+ # of a table.
+ # Here are some things that make sense to mark as post deployment:
+ # - Large data migrations. Typically we want these to be run manually so that they can be
+ # monitored and not block the deploy for a long period of time while they run.
+ # - Adding indexes to large tables. Since this can take a long time, we'd generally prefer to
+ # run this outside deployments so that we don't block them. Note that while adding an index
+ # is a schema change, it's completely safe to run the operation after the code has deployed.
+ # Once deployed, run these manually via: https://develop.sentry.dev/database-migrations/#migration-deployment
+
+ is_post_deployment = True
+
+ dependencies = [
+ ("sentry", "0752_fix_substatus_for_unresolved_groups"),
+ ]
+
+ operations = [
+ migrations.RunPython(
+ backfill_substatus_for_ignored_groups,
+ migrations.RunPython.noop,
+ hints={"tables": ["sentry_groupedmessage"]},
+ ),
+ ]

tests/sentry/migrations/test_0753_fix_substatus_for_ignored_groups.py

@@ -0,0 +1,102 @@
+from sentry.models.activity import Activity
+from sentry.models.group import Group, GroupStatus
+from sentry.models.groupsnooze import GroupSnooze
+from sentry.models.organization import Organization
+from sentry.testutils.cases import TestMigrations
+from sentry.types.activity import ActivityType
+from sentry.types.group import GroupSubStatus
+
+
+class FixSubstatusForIgnoreedGroupsTest(TestMigrations):
+ migrate_from = "0752_fix_substatus_for_unresolved_groups"
+ migrate_to = "0753_fix_substatus_for_ignored_groups"
+
+ def setup_before_migration(self, app):
+ self.organization = Organization.objects.create(name="test", slug="test")
+ self.project = self.create_project(organization=self.organization)
+ self.do_not_update = Group.objects.create(
+ project=self.project,
+ status=GroupStatus.IGNORED,
+ substatus=GroupSubStatus.UNTIL_ESCALATING,
+ )
+
+ self.ignored_until_condition_met = Group.objects.create(
+ project=self.project,
+ status=GroupStatus.IGNORED,
+ )
+ # .update() skips calling the pre_save checks which requires a substatus
+ self.ignored_until_condition_met.update(substatus=None)
+ self.ignored_until_condition_met.refresh_from_db()
+ assert self.ignored_until_condition_met.substatus is None
+ Activity.objects.create(
+ group=self.ignored_until_condition_met,
+ project=self.project,
+ type=ActivityType.SET_IGNORED.value,
+ data={"ignoreCount": 10},
+ )
+
+ self.ignored_until_condition_met_no_activity = Group.objects.create(
+ project=self.project,
+ status=GroupStatus.IGNORED,
+ )
+ self.ignored_until_condition_met_no_activity.update(substatus=None)
+ assert self.ignored_until_condition_met_no_activity.substatus is None
+ Activity.objects.create(
+ group=self.ignored_until_condition_met_no_activity,
+ project=self.project,
+ type=ActivityType.SET_IGNORED.value,
+ data={
+ "ignoreCount": None,
+ "ignoreDuration": None,
+ "ignoreUntil": None,
+ "ignoreUserCount": None,
+ "ignoreUserWindow": None,
+ "ignoreWindow": None,
+ "ignoreUntilEscalating": None,
+ },
+ )
+ GroupSnooze.objects.create(
+ group=self.ignored_until_condition_met_no_activity,
+ count=10,
+ )
+
+ self.ignored_until_escalating = Group.objects.create(
+ project=self.project,
+ status=GroupStatus.IGNORED,
+ )
+ # .update() skips calling the pre_save checks which requires a substatus
+ self.ignored_until_escalating.update(substatus=None)
+ self.ignored_until_escalating.refresh_from_db()
+ assert self.ignored_until_escalating.substatus is None
+ Activity.objects.create(
+ group=self.ignored_until_escalating,
+ project=self.project,
+ type=ActivityType.SET_IGNORED.value,
+ data={"ignoreUntilEscalating": True},
+ )
+
+ self.ignored_forever = Group.objects.create(
+ project=self.project,
+ status=GroupStatus.IGNORED,
+ )
+ self.ignored_forever.update(substatus=None)
+ assert self.ignored_forever.substatus is None
+
+ def test(self):
+ self.do_not_update.refresh_from_db()
+ assert self.do_not_update.substatus == GroupSubStatus.UNTIL_ESCALATING
+
+ self.ignored_until_condition_met.refresh_from_db()
+ assert self.ignored_until_condition_met.substatus == GroupSubStatus.UNTIL_CONDITION_MET
+
+ self.ignored_until_condition_met_no_activity.refresh_from_db()
+ assert (
+ self.ignored_until_condition_met_no_activity.substatus
+ == GroupSubStatus.UNTIL_CONDITION_MET
+ )
+
+ self.ignored_until_escalating.refresh_from_db()
+ assert self.ignored_until_escalating.substatus == GroupSubStatus.UNTIL_ESCALATING
+
+ self.ignored_forever.refresh_from_db()
+ assert self.ignored_forever.substatus == GroupSubStatus.FOREVER