chore(seer): Track rate limited outcomes (#94358)

Emits a rate limited outcome when Seer scanner or auto-Autofix are rate
limited. Refactors the rate limiting utils a bit to encapsulate the
logging and outcome logic in one place.

Author: roaga

src/sentry/api/endpoints/group_autofix_setup_check.py

@@ -138,7 +138,6 @@ def get(self, request: Request, group: Group) -> Response:
         if not user_acknowledgement:  # If the user has acknowledged, the org must have too.
             org_acknowledgement = get_seer_org_acknowledgement(org_id=org.id)
 
-        # TODO return BOTH trial status and autofix quota
         has_autofix_quota: bool = quotas.backend.has_available_reserved_budget(
             org_id=org.id, data_category=DataCategory.SEER_AUTOFIX
         )

src/sentry/autofix/utils.py

@@ -1,5 +1,6 @@
-import datetime
 import enum
+import logging
+from datetime import UTC, datetime
 from typing import TypedDict
 
 import orjson
@@ -8,13 +9,17 @@
 from pydantic import BaseModel
 
 from sentry import features, options, ratelimits
+from sentry.constants import DataCategory
 from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.models.organization import Organization
 from sentry.models.project import Project
 from sentry.models.repository import Repository
 from sentry.seer.seer_utils import AutofixAutomationTuningSettings
 from sentry.seer.signed_seer_api import sign_with_seer_secret
 from sentry.utils import json
+from sentry.utils.outcomes import Outcome, track_outcome
+
+logger = logging.getLogger(__name__)
 
 
 class AutofixIssue(TypedDict):
@@ -51,7 +56,7 @@ class CodebaseState(BaseModel):
 class AutofixState(BaseModel):
     run_id: int
     request: AutofixRequest
-    updated_at: datetime.datetime
+    updated_at: datetime
     status: AutofixStatus
     actor_ids: list[str] | None = None
     codebases: dict[str, CodebaseState] = {}
@@ -162,20 +167,20 @@ class SeerAutomationSource(enum.Enum):
     POST_PROCESS = "post_process"
 
 
-def is_seer_scanner_rate_limited(
-    project: Project, organization: Organization
-) -> tuple[bool, int, int]:
+def is_seer_scanner_rate_limited(project: Project, organization: Organization) -> bool:
     """
     Check if Seer Scanner automation is rate limited for a given project and organization.
+    Calling this method increments the counter used to enforce the rate limit, and tracks rate limited outcomes.
+
+    Args:
+        project: The project to check.
+        organization: The organization to check.
 
     Returns:
-        tuple[bool, int, int]:
-            - is_rate_limited: Whether the seer scanner is rate limited.
-            - current: The current number of seer scanner runs.
-            - limit: The limit for seer scanner runs.
+        bool: Whether the seer scanner is rate limited.
     """
     if features.has("organizations:unlimited-auto-triggered-autofix-runs", organization):
-        return False, 0, 0
+        return False
 
     limit = options.get("seer.max_num_scanner_autotriggered_per_minute", 50)
     is_rate_limited, current, _ = ratelimits.backend.is_limited_with_value(
@@ -184,7 +189,26 @@ def is_seer_scanner_rate_limited(
         limit=limit,
         window=60,  # 1 minute
     )
-    return is_rate_limited, current, limit
+    if is_rate_limited:
+        logger.info(
+            "Seer scanner auto-trigger rate limit hit",
+            extra={
+                "org_slug": organization.slug,
+                "project_slug": project.slug,
+                "scanner_run_count": current,
+                "scanner_run_limit": limit,
+            },
+        )
+        track_outcome(
+            org_id=organization.id,
+            project_id=project.id,
+            key_id=None,
+            outcome=Outcome.RATE_LIMITED,
+            reason="rate_limited",
+            timestamp=datetime.now(UTC),
+            category=DataCategory.SEER_SCANNER,
+        )
+    return is_rate_limited
 
 
 AUTOFIX_AUTOTRIGGED_RATE_LIMIT_OPTION_MULTIPLIERS = {
@@ -200,18 +224,20 @@ def is_seer_scanner_rate_limited(
 
 def is_seer_autotriggered_autofix_rate_limited(
     project: Project, organization: Organization
-) -> tuple[bool, int, int]:
+) -> bool:
     """
     Check if Seer Autofix automation is rate limited for a given project and organization.
+    Calling this method increments the counter used to enforce the rate limit, and tracks rate limited outcomes.
+
+    Args:
+        project: The project to check.
+        organization: The organization to check.
 
     Returns:
-        tuple[bool, int, int]:
-            - is_rate_limited: Whether Autofix is rate limited.
-            - current: The current number of Autofix runs.
-            - limit: The limit for Autofix runs.
+        bool: Whether Autofix is rate limited.
     """
     if features.has("organizations:unlimited-auto-triggered-autofix-runs", organization):
-        return False, 0, 0
+        return False
 
     limit = options.get("seer.max_num_autofix_autotriggered_per_hour", 20)
 
@@ -228,4 +254,23 @@ def is_seer_autotriggered_autofix_rate_limited(
         limit=limit,
         window=60 * 60,  # 1 hour
     )
-    return is_rate_limited, current, limit
+    if is_rate_limited:
+        logger.info(
+            "Autofix auto-trigger rate limit hit",
+            extra={
+                "auto_run_count": current,
+                "auto_run_limit": limit,
+                "org_slug": organization.slug,
+                "project_slug": project.slug,
+            },
+        )
+        track_outcome(
+            org_id=organization.id,
+            project_id=project.id,
+            key_id=None,
+            outcome=Outcome.RATE_LIMITED,
+            reason="rate_limited",
+            timestamp=datetime.now(UTC),
+            category=DataCategory.SEER_AUTOFIX,
+        )
+    return is_rate_limited

src/sentry/integrations/utils/issue_summary_for_alerts.py

@@ -31,18 +31,7 @@ def fetch_issue_summary(group: Group) -> dict[str, Any] | None:
     if not get_seer_org_acknowledgement(org_id=group.organization.id):
         return None
 
-    is_rate_limited, current, limit = is_seer_scanner_rate_limited(project, group.organization)
-    if is_rate_limited:
-        logger.warning(
-            "Seer scanner auto-trigger rate limit hit",
-            extra={
-                "org_slug": group.organization.slug,
-                "project_slug": project.slug,
-                "group_id": group.id,
-                "scanner_run_count": current,
-                "scanner_run_limit": limit,
-            },
-        )
+    if is_seer_scanner_rate_limited(project, group.organization):
         return None
 
     from sentry import quotas

src/sentry/seer/issue_summary.py

@@ -318,20 +318,8 @@ def _run_automation(
     if autofix_state:
         return  # already have an autofix on this issue
 
-    is_rate_limited, current, limit = is_seer_autotriggered_autofix_rate_limited(
-        group.project, group.organization
-    )
+    is_rate_limited = is_seer_autotriggered_autofix_rate_limited(group.project, group.organization)
     if is_rate_limited:
-        logger.warning(
-            "Autofix auto-trigger rate limit hit",
-            extra={
-                "group_id": group.id,
-                "auto_run_count": current,
-                "auto_run_limit": limit,
-                "org_slug": group.organization.slug,
-                "project_slug": group.project.slug,
-            },
-        )
         return
 
     _trigger_autofix_task.delay(

src/sentry/tasks/post_process.py

@@ -1617,18 +1617,7 @@ def kick_off_seer_automation(job: PostProcessJob) -> None:
 
     from sentry.autofix.utils import is_seer_scanner_rate_limited
 
-    is_rate_limited, current, limit = is_seer_scanner_rate_limited(project, group.organization)
-    if is_rate_limited:
-        logger.warning(
-            "Seer scanner auto-trigger rate limit hit",
-            extra={
-                "org_slug": group.organization.slug,
-                "project_slug": project.slug,
-                "group_id": group.id,
-                "scanner_run_count": current,
-                "scanner_run_limit": limit,
-            },
-        )
+    if is_seer_scanner_rate_limited(project, group.organization):
         return
 
     start_seer_automation.delay(group.id)

tests/sentry/autofix/test_utils.py

@@ -191,48 +191,47 @@ def test_get_autofix_state_http_error(self, mock_post):
 
 class TestAutomationRateLimiting(TestCase):
     @with_feature("organizations:unlimited-auto-triggered-autofix-runs")
-    def test_scanner_rate_limited_with_unlimited_flag(self):
+    @patch("sentry.autofix.utils.track_outcome")
+    def test_scanner_rate_limited_with_unlimited_flag(self, mock_track_outcome):
         """Test scanner rate limiting bypassed with unlimited feature flag"""
         project = self.create_project()
         organization = project.organization
 
-        is_rate_limited, current, limit = is_seer_scanner_rate_limited(project, organization)
+        is_rate_limited = is_seer_scanner_rate_limited(project, organization)
 
         assert is_rate_limited is False
-        assert current == 0
-        assert limit == 0
+        mock_track_outcome.assert_not_called()
 
     @patch("sentry.autofix.utils.ratelimits.backend.is_limited_with_value")
-    def test_scanner_rate_limited_logic(self, mock_is_limited):
+    @patch("sentry.autofix.utils.track_outcome")
+    def test_scanner_rate_limited_logic(self, mock_track_outcome, mock_is_limited):
         """Test scanner rate limiting logic"""
         project = self.create_project()
         organization = project.organization
 
         mock_is_limited.return_value = (True, 45, None)
 
         with self.options({"seer.max_num_scanner_autotriggered_per_minute": 50}):
-            is_rate_limited, current, limit = is_seer_scanner_rate_limited(project, organization)
+            is_rate_limited = is_seer_scanner_rate_limited(project, organization)
 
         assert is_rate_limited is True
-        assert current == 45
-        assert limit == 50
+        mock_track_outcome.assert_called_once()
 
     @with_feature("organizations:unlimited-auto-triggered-autofix-runs")
-    def test_autofix_rate_limited_with_unlimited_flag(self):
+    @patch("sentry.autofix.utils.track_outcome")
+    def test_autofix_rate_limited_with_unlimited_flag(self, mock_track_outcome):
         """Test autofix rate limiting bypassed with unlimited feature flag"""
         project = self.create_project()
         organization = project.organization
 
-        is_rate_limited, current, limit = is_seer_autotriggered_autofix_rate_limited(
-            project, organization
-        )
+        is_rate_limited = is_seer_autotriggered_autofix_rate_limited(project, organization)
 
         assert is_rate_limited is False
-        assert current == 0
-        assert limit == 0
+        mock_track_outcome.assert_not_called()
 
     @patch("sentry.autofix.utils.ratelimits.backend.is_limited_with_value")
-    def test_autofix_rate_limited_logic(self, mock_is_limited):
+    @patch("sentry.autofix.utils.track_outcome")
+    def test_autofix_rate_limited_logic(self, mock_track_outcome, mock_is_limited):
         """Test autofix rate limiting logic"""
         project = self.create_project()
         organization = project.organization
@@ -242,13 +241,10 @@ def test_autofix_rate_limited_logic(self, mock_is_limited):
         mock_is_limited.return_value = (True, 19, None)
 
         with self.options({"seer.max_num_autofix_autotriggered_per_hour": 20}):
-            is_rate_limited, current, limit = is_seer_autotriggered_autofix_rate_limited(
-                project, organization
-            )
+            is_rate_limited = is_seer_autotriggered_autofix_rate_limited(project, organization)
 
         assert is_rate_limited is True
-        assert current == 19
-        assert limit == 20
+        mock_track_outcome.assert_called_once()
 
     @patch("sentry.autofix.utils.ratelimits.backend.is_limited_with_value")
     def test_autofix_rate_limit_multiplication_logic(self, mock_is_limited):

tests/sentry/tasks/test_post_process.py

@@ -2698,7 +2698,7 @@ def test_rate_limit_only_checked_after_all_other_checks_pass(
         """Test that rate limit check only happens after all other checks pass"""
         mock_get_seer_org_acknowledgement.return_value = True
         mock_has_budget.return_value = True
-        mock_is_rate_limited.return_value = (False, 0, 100)  # Not rate limited
+        mock_is_rate_limited.return_value = False
 
         self.project.update_option("sentry:seer_scanner_automation", True)
         event = self.create_event(