fix(teamStats): Fix timeouts on Unresolved Issues graph (#95715)

The Unresolved Issues graph is currently hanging, then eventually
erroring out. Investigating, it seems like it's the DB query timing out.
Signs point to that coming from the large number of subqueries involved
in the `GroupHistory` query.

Those subqueries exist because `GroupHistory` may contain multiple rows
with the same `status` for any given `group_id` (or with different
`status` in our same "state" category), so we need to deduplicate to
only rows that change between an open or closed state.

The current solution (subqueries) is inefficient on large # of
group_ids, which can lead to timeouts.

Another option would be to use `DISTINCT ON` queries to ensure we only
get one row per each `(group_id, open/closed state)` pair. Unfortunately
Django does not currently support `annotate()` and `distinct(fields)`
together.

Instead, what we do here is pull additional data and do the
deduplication logic in Python.

Test plan: loads without error in my sandbox.

---------

Co-authored-by: Charles Paul <charlespaul@R4KCFH7MHY.local>

Author: thetruecpaul

src/sentry/api/endpoints/team_all_unresolved_issues.py

@@ -1,10 +1,11 @@
 import copy
 import datetime
+from collections import defaultdict
 from datetime import timedelta
 from itertools import chain
 
-from django.db.models import Count, OuterRef, Q, QuerySet, Subquery
-from django.db.models.functions import Coalesce, TruncDay
+from django.db.models import Case, Count, Q, QuerySet, Value, When
+from django.db.models.functions import TruncDay
 from rest_framework.request import Request
 from rest_framework.response import Response
 
@@ -53,22 +54,6 @@ def calculate_unresolved_counts(
     group_history_environment_filter = (
         Q(group__groupenvironment__environment_id=environment_id) if environment_id else Q()
     )
-    prev_status_sub_qs = Coalesce(
-        Subquery(
-            GroupHistory.objects.filter(
-                group_id=OuterRef("group_id"),
-                date_added__lt=OuterRef("date_added"),
-                status__in=OPEN_STATUSES + CLOSED_STATUSES,
-            )
-            .order_by("-id")
-            .values("status")[:1]
-        ),
-        -1,
-    )
-    dedupe_status_filter = Q(
-        (~Q(prev_status__in=OPEN_STATUSES) & Q(status__in=OPEN_STATUSES))
-        | (~Q(prev_status__in=CLOSED_STATUSES) & Q(status__in=CLOSED_STATUSES))
-    )
 
     # Grab the historical data bucketed by day
     new_issues = (
@@ -80,19 +65,62 @@ def calculate_unresolved_counts(
         .annotate(open=Count("id"))
     )
 
+    # Pull extra data to do deduplication in Python. (Inefficient to do in SQL via subqueries
+    # (see ISWF-549); cannot do via DISTINCT ON state because of Django limitations.)
     bucketed_issues = (
         GroupHistory.objects.filter_to_team(team)
-        .filter(group_history_environment_filter, date_added__gte=start, date_added__lte=end)
-        .annotate(bucket=TruncDay("date_added"), prev_status=prev_status_sub_qs)
-        .filter(dedupe_status_filter)
-        .order_by("bucket")
-        .values("project", "bucket")
+        .filter(
+            group_history_environment_filter,
+            date_added__gte=start,
+            date_added__lte=end,
+            status__in=OPEN_STATUSES + CLOSED_STATUSES,
+        )
         .annotate(
-            open=Count("id", filter=Q(status__in=OPEN_STATUSES)),
-            closed=Count("id", filter=Q(status__in=CLOSED_STATUSES)),
+            bucket=TruncDay("date_added"),
+            state=Case(
+                When(status__in=OPEN_STATUSES, then=Value("open")),
+                When(status__in=CLOSED_STATUSES, then=Value("closed")),
+                default=Value("other"),
+            ),
+        )
+        .order_by(
+            "group_id",
+            "bucket",
+            "id",
         )
+        .values("project", "group_id", "bucket", "state")[:200_000]
     )
 
+    most_recent_group_state: defaultdict[str, str] = defaultdict(lambda: "other")
+    # Project => Bucket => State => Count
+    deduping_map: defaultdict[str, defaultdict[str, defaultdict[str, int]]] = defaultdict(
+        lambda: defaultdict(lambda: defaultdict(int))
+    )
+    for r in bucketed_issues:
+        # Don't process the row if it doesn't set the group to open or closed.
+        if r["state"] == "other":
+            continue
+
+        # Don't process the row if it doesn't change the state.
+        if r["state"] == most_recent_group_state[r["group_id"]]:
+            continue
+
+        deduping_map[r["project"]][r["bucket"]][r["state"]] += 1
+        most_recent_group_state[r["group_id"]] = r["state"]
+
+    deduped_bucketed_issues = []
+    for p in deduping_map.keys():
+        bucket_counts = deduping_map[p]
+        for b in bucket_counts.keys():
+            deduped_bucketed_issues.append(
+                {
+                    "project": p,
+                    "bucket": b,
+                    "open": bucket_counts[b]["open"],
+                    "closed": bucket_counts[b]["closed"],
+                }
+            )
+
     current_day, date_series_dict = start, {}
     while current_day < end:
         date_series_dict[current_day.isoformat()] = {"open": 0, "closed": 0}
@@ -101,7 +129,7 @@ def calculate_unresolved_counts(
     agg_project_precounts = {
         project.id: copy.deepcopy(date_series_dict) for project in project_list
     }
-    for r in chain(bucketed_issues, new_issues):
+    for r in chain(deduped_bucketed_issues, new_issues):
         bucket = agg_project_precounts[r["project"]][r["bucket"].isoformat()]
         bucket["open"] += r.get("open", 0)
         bucket["closed"] += r.get("closed", 0)

tests/sentry/api/endpoints/test_team_all_unresolved_issues.py

@@ -1,6 +1,5 @@
 from datetime import datetime, timedelta, timezone
 
-import pytest
 from django.utils.timezone import now
 
 from sentry.models.group import GroupStatus
@@ -15,7 +14,6 @@
 class TeamIssueBreakdownTest(APITestCase):
     endpoint = "sentry-api-0-team-all-unresolved-issues"
 
-    @pytest.mark.xfail(reason="flakey")
     def test_status_format(self):
         project1 = self.create_project(teams=[self.team])
         group1_1 = self.create_group(project=project1, first_seen=before_now(days=40))