chore(seer): optimize scanner rate limiting (#93928)

- Bump scanner rate limit to 2500 per hour. Since we now scan existing
issues as well, and since we double-count in slack alerts, we're going
to be hitting the rate limit a lot more. This is still respectable for
budgets, capping at $7.50.
- Only increment and check rate limit _after_ we pass the quota check
and are about to start a scanner task. Right now we're logging rate
limit exceeded errors for customers without any quota.

Author: roaga

src/sentry/autofix/utils.py

@@ -176,7 +176,7 @@ def is_seer_scanner_rate_limited(
     if features.has("organizations:unlimited-auto-triggered-autofix-runs", organization):
         return False, 0, 0
 
-    limit = options.get("seer.max_num_scanner_autotriggered_per_hour", 1000)
+    limit = options.get("seer.max_num_scanner_autotriggered_per_hour", 2500)
     is_rate_limited, current, _ = ratelimits.backend.is_limited_with_value(
         project=project,
         key="seer.scanner.auto_triggered",

src/sentry/options/defaults.py

@@ -603,7 +603,7 @@
 register("seer.max_num_autofix_autotriggered_per_hour", default=20, flags=FLAG_AUTOMATOR_MODIFIABLE)
 # Seer Scanner Auto-trigger rate (max number of scans auto-triggered per project per hour)
 register(
-    "seer.max_num_scanner_autotriggered_per_hour", default=1000, flags=FLAG_AUTOMATOR_MODIFIABLE
+    "seer.max_num_scanner_autotriggered_per_hour", default=2500, flags=FLAG_AUTOMATOR_MODIFIABLE
 )
 
 # Codecov Integration

src/sentry/tasks/post_process.py

@@ -1600,6 +1600,15 @@ def kick_off_seer_automation(job: PostProcessJob) -> None:
     if not seer_enabled:
         return
 
+    from sentry import quotas
+    from sentry.constants import DataCategory
+
+    has_budget: bool = quotas.backend.has_available_reserved_budget(
+        org_id=group.organization.id, data_category=DataCategory.SEER_SCANNER
+    )
+    if not has_budget:
+        return
+
     from sentry.autofix.utils import is_seer_scanner_rate_limited
 
     is_rate_limited, current, limit = is_seer_scanner_rate_limited(project, group.organization)
@@ -1613,15 +1622,6 @@ def kick_off_seer_automation(job: PostProcessJob) -> None:
         logger.error("Seer scanner auto-trigger rate limit hit")
         return
 
-    from sentry import quotas
-    from sentry.constants import DataCategory
-
-    has_budget: bool = quotas.backend.has_available_reserved_budget(
-        org_id=group.organization.id, data_category=DataCategory.SEER_SCANNER
-    )
-    if not has_budget:
-        return
-
     start_seer_automation.delay(group.id)
 
 

tests/sentry/tasks/test_post_process.py

@@ -2682,6 +2682,102 @@ def test_kick_off_seer_automation_runs_with_missing_summary_cache(
 
         mock_start_seer_automation.assert_called_once_with(group.id)
 
+    @patch("sentry.autofix.utils.is_seer_scanner_rate_limited")
+    @patch("sentry.quotas.backend.has_available_reserved_budget")
+    @patch("sentry.seer.seer_setup.get_seer_org_acknowledgement")
+    @patch("sentry.tasks.autofix.start_seer_automation.delay")
+    @with_feature("organizations:gen-ai-features")
+    @with_feature("organizations:trigger-autofix-on-issue-summary")
+    def test_rate_limit_only_checked_after_all_other_checks_pass(
+        self,
+        mock_start_seer_automation,
+        mock_get_seer_org_acknowledgement,
+        mock_has_budget,
+        mock_is_rate_limited,
+    ):
+        """Test that rate limit check only happens after all other checks pass"""
+        mock_get_seer_org_acknowledgement.return_value = True
+        mock_has_budget.return_value = True
+        mock_is_rate_limited.return_value = (False, 0, 100)  # Not rate limited
+
+        self.project.update_option("sentry:seer_scanner_automation", True)
+        event = self.create_event(
+            data={"message": "testing"},
+            project_id=self.project.id,
+        )
+
+        # Test 1: When all checks pass, rate limit should be checked
+        self.call_post_process_group(
+            is_new=True,
+            is_regression=False,
+            is_new_group_environment=True,
+            event=event,
+        )
+        mock_is_rate_limited.assert_called_once_with(event.project, event.group.organization)
+        mock_start_seer_automation.assert_called_once_with(event.group.id)
+
+        mock_is_rate_limited.reset_mock()
+        mock_start_seer_automation.reset_mock()
+
+        # Test 2: When seer org acknowledgement fails, rate limit should NOT be checked
+        mock_get_seer_org_acknowledgement.return_value = False
+
+        event2 = self.create_event(
+            data={"message": "testing 2"},
+            project_id=self.project.id,
+        )
+
+        self.call_post_process_group(
+            is_new=True,
+            is_regression=False,
+            is_new_group_environment=True,
+            event=event2,
+        )
+        mock_is_rate_limited.assert_not_called()
+        mock_start_seer_automation.assert_not_called()
+
+        mock_is_rate_limited.reset_mock()
+        mock_start_seer_automation.reset_mock()
+        mock_get_seer_org_acknowledgement.return_value = True  # Reset to success
+
+        # Test 3: When budget check fails, rate limit should NOT be checked
+        mock_has_budget.return_value = False
+
+        event3 = self.create_event(
+            data={"message": "testing 3"},
+            project_id=self.project.id,
+        )
+
+        self.call_post_process_group(
+            is_new=True,
+            is_regression=False,
+            is_new_group_environment=True,
+            event=event3,
+        )
+        mock_is_rate_limited.assert_not_called()
+        mock_start_seer_automation.assert_not_called()
+
+        mock_is_rate_limited.reset_mock()
+        mock_start_seer_automation.reset_mock()
+        mock_has_budget.return_value = True  # Reset to success
+
+        # Test 4: When project option is disabled, rate limit should NOT be checked
+        self.project.update_option("sentry:seer_scanner_automation", False)
+
+        event4 = self.create_event(
+            data={"message": "testing 4"},
+            project_id=self.project.id,
+        )
+
+        self.call_post_process_group(
+            is_new=True,
+            is_regression=False,
+            is_new_group_environment=True,
+            event=event4,
+        )
+        mock_is_rate_limited.assert_not_called()
+        mock_start_seer_automation.assert_not_called()
+
 
 class PostProcessGroupErrorTest(
     TestCase,