chore(autofix): Dynamic auto-autofix rate limit (#94256)

Increases the rate limit for auto-triggered Autofix as your automation
setting becomes more strict. The goal of this logic is to enable high
usage for large orgs with reasonable settings, while protecting small
orgs with extreme settings from surprise bills.

Author: roaga

src/sentry/autofix/utils.py

@@ -186,6 +186,17 @@ def is_seer_scanner_rate_limited(
     return is_rate_limited, current, limit
 
 
+AUTOFIX_AUTOTRIGGED_RATE_LIMIT_OPTION_MULTIPLIERS = {
+    "off": 5,
+    "super_low": 5,
+    "low": 4,
+    "medium": 3,
+    "high": 2,
+    "always": 1,
+    None: 1,  # default if option is not set
+}
+
+
 def is_seer_autotriggered_autofix_rate_limited(
     project: Project, organization: Organization
 ) -> tuple[bool, int, int]:
@@ -202,6 +213,14 @@ def is_seer_autotriggered_autofix_rate_limited(
         return False, 0, 0
 
     limit = options.get("seer.max_num_autofix_autotriggered_per_hour", 20)
+
+    # The more selective automation is, the higher the limit we allow.
+    # This is to protect projects with extreme settings from starting too many runs
+    # while allowing big projects with reasonable settings to run more often.
+    option = project.get_option("sentry:autofix_automation_tuning")
+    multiplier = AUTOFIX_AUTOTRIGGED_RATE_LIMIT_OPTION_MULTIPLIERS.get(option, 1)
+    limit *= multiplier
+
     is_rate_limited, current, _ = ratelimits.backend.is_limited_with_value(
         project=project,
         key="autofix.auto_triggered",

tests/sentry/autofix/test_utils.py

@@ -5,6 +5,7 @@
 from django.conf import settings
 
 from sentry.autofix.utils import (
+    AUTOFIX_AUTOTRIGGED_RATE_LIMIT_OPTION_MULTIPLIERS,
     AutofixState,
     AutofixStatus,
     get_autofix_repos_from_project_code_mappings,
@@ -236,6 +237,8 @@ def test_autofix_rate_limited_logic(self, mock_is_limited):
         project = self.create_project()
         organization = project.organization
 
+        project.update_option("sentry:autofix_automation_tuning", None)
+
         mock_is_limited.return_value = (True, 19, None)
 
         with self.options({"seer.max_num_autofix_autotriggered_per_hour": 20}):
@@ -246,3 +249,24 @@ def test_autofix_rate_limited_logic(self, mock_is_limited):
         assert is_rate_limited is True
         assert current == 19
         assert limit == 20
+
+    @patch("sentry.autofix.utils.ratelimits.backend.is_limited_with_value")
+    def test_autofix_rate_limit_multiplication_logic(self, mock_is_limited):
+        """Test that the limit is multiplied correctly based on project option"""
+        project = self.create_project()
+        organization = project.organization
+        mock_is_limited.return_value = (False, 0, None)
+
+        base_limit = 20
+
+        for option, multiplier in AUTOFIX_AUTOTRIGGED_RATE_LIMIT_OPTION_MULTIPLIERS.items():
+            with self.options({"seer.max_num_autofix_autotriggered_per_hour": base_limit}):
+                project.update_option("sentry:autofix_automation_tuning", option)
+                is_seer_autotriggered_autofix_rate_limited(project, organization)
+                expected_limit = base_limit * multiplier
+                mock_is_limited.assert_called_with(
+                    project=project,
+                    key="autofix.auto_triggered",
+                    limit=expected_limit,
+                    window=60 * 60,
+                )