ref(escalating-issues): Auto-transition tasks should update up to 500…

…_000 groups per minute (#56168)

## Objective:

The current implementation of the auto-transition tasks is leading to
spiky memory pressure on RabbitMQ. This is partly due to the hot shards
of big orgs with a lot of groups. The alternative approach is to
consistently send x number of messages of groups older than 7 days. No
need to care about org or project bc all those groups have the same
status and substatus changes. This iteration sends up to 50 child tasks
with each task updating 10_000 groups. We will increase the number of
tasks if the backlog of groups_to_be_updated is increasing.

src/sentry/conf/server.py

@@ -1131,8 +1131,8 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
  },
  "schedule_auto_transition_to_ongoing": {
  "task": "sentry.tasks.schedule_auto_transition_to_ongoing",
- # Run job every 10 minutes
- "schedule": crontab(minute="*/10"),
+ # Run job every minute
+ "schedule": crontab(minute="*/1"),
  "options": {"expires": 3600},
  },
  "github_comment_reactions": {

src/sentry/issues/ongoing.py

@@ -10,12 +10,12 @@
 def bulk_transition_group_to_ongoing(
  from_status: int,
  from_substatus: int,
- groups: List[Group],
+ group_ids: List[int],
  activity_data: Optional[Mapping[str, Any]] = None,
 ) -> None:
  # make sure we don't update the Group when its already updated by conditionally updating the Group
  groups_to_transistion = Group.objects.filter(
- id__in=[group.id for group in groups], status=from_status, substatus=from_substatus
+ id__in=group_ids, status=from_status, substatus=from_substatus
  )
 
  Group.objects.update_group_status(
@@ -31,7 +31,7 @@ def bulk_transition_group_to_ongoing(
  group.status = GroupStatus.UNRESOLVED
  group.substatus = GroupSubStatus.ONGOING
 
- bulk_remove_groups_from_inbox(groups)
+ bulk_remove_groups_from_inbox(groups_to_transistion)
 
  for group in groups_to_transistion:
  post_save.send_robust(

src/sentry/issues/update_inbox.py

@@ -54,7 +54,7 @@ def update_inbox(
  bulk_transition_group_to_ongoing(
  group.status,
  group.substatus,
- [group],
+ [group.id],
  activity_data={"manually": True},
  )