Merge branch 'master' into constantinius/ref/analytics/tet-831

Author: constantinius

src/sentry/api/endpoints/project_details.py

@@ -36,11 +36,7 @@
 from sentry.deletions.models.scheduleddeletion import RegionScheduledDeletion
 from sentry.dynamic_sampling import get_supported_biases_ids, get_user_biases
 from sentry.dynamic_sampling.types import DynamicSamplingMode
-from sentry.dynamic_sampling.utils import (
-    has_custom_dynamic_sampling,
-    has_dynamic_sampling,
-    has_dynamic_sampling_minimum_sample_rate,
-)
+from sentry.dynamic_sampling.utils import has_custom_dynamic_sampling, has_dynamic_sampling
 from sentry.grouping.enhancer import Enhancements
 from sentry.grouping.enhancer.exceptions import InvalidEnhancerConfig
 from sentry.grouping.fingerprinting import FingerprintingRules, InvalidFingerprintingConfig
@@ -131,7 +127,6 @@ class ProjectMemberSerializer(serializers.Serializer):
         "copy_from_project",
         "targetSampleRate",
         "dynamicSamplingBiases",
-        "dynamicSamplingMinimumSampleRate",
         "tempestFetchScreenshots",
         "tempestFetchDumps",
         "autofixAutomationTuning",
@@ -225,7 +220,6 @@ class ProjectAdminSerializer(ProjectMemberSerializer):
     copy_from_project = serializers.IntegerField(required=False)
     targetSampleRate = serializers.FloatField(required=False, min_value=0, max_value=1)
     dynamicSamplingBiases = DynamicSamplingBiasSerializer(required=False, many=True)
-    dynamicSamplingMinimumSampleRate = serializers.BooleanField(required=False)
     tempestFetchScreenshots = serializers.BooleanField(required=False)
     tempestFetchDumps = serializers.BooleanField(required=False)
     autofixAutomationTuning = serializers.ChoiceField(
@@ -434,15 +428,6 @@ def validate_targetSampleRate(self, value):
 
         return value
 
-    def validate_dynamicSamplingMinimumSampleRate(self, value):
-        organization = self.context["project"].organization
-        actor = self.context["request"].user
-        if not has_dynamic_sampling_minimum_sample_rate(organization, actor=actor):
-            raise serializers.ValidationError(
-                "Organization does not have the dynamic sampling minimum sample rate feature enabled."
-            )
-        return value
-
     def validate_tempestFetchScreenshots(self, value):
         organization = self.context["project"].organization
         actor = self.context["request"].user
@@ -773,14 +758,6 @@ def put(self, request: Request, project) -> Response:
                 changed_proj_settings["sentry:dynamic_sampling_biases"] = result[
                     "dynamicSamplingBiases"
                 ]
-        if result.get("dynamicSamplingMinimumSampleRate") is not None:
-            if project.update_option(
-                "sentry:dynamic_sampling_minimum_sample_rate",
-                result["dynamicSamplingMinimumSampleRate"],
-            ):
-                changed_proj_settings["sentry:dynamic_sampling_minimum_sample_rate"] = result[
-                    "dynamicSamplingMinimumSampleRate"
-                ]
 
         if result.get("autofixAutomationTuning") is not None:
             if project.update_option(

src/sentry/api/serializers/models/project.py

@@ -25,7 +25,6 @@
 from sentry.dynamic_sampling.utils import (
     has_custom_dynamic_sampling,
     has_dynamic_sampling,
-    has_dynamic_sampling_minimum_sample_rate,
     is_project_mode_sampling,
 )
 from sentry.eventstore.models import DEFAULT_SUBJECT_TEMPLATE
@@ -950,7 +949,6 @@ class DetailedProjectResponse(ProjectWithTeamResponseDict):
     relayPiiConfig: str | None
     builtinSymbolSources: list[str]
     dynamicSamplingBiases: list[dict[str, str | bool]]
-    dynamicSamplingMinimumSampleRate: bool
     eventProcessing: dict[str, bool]
     symbolSources: str
     isDynamicallySampled: bool
@@ -1101,9 +1099,6 @@ def serialize(
             "dynamicSamplingBiases": self.get_value_with_default(
                 attrs, "sentry:dynamic_sampling_biases"
             ),
-            "dynamicSamplingMinimumSampleRate": self.get_value_with_default(
-                attrs, "sentry:dynamic_sampling_minimum_sample_rate"
-            ),
             "eventProcessing": {
                 "symbolicationDegraded": False,
             },
@@ -1123,11 +1118,6 @@ def serialize(
             )
             data["tempestFetchDumps"] = attrs["options"].get("sentry:tempest_fetch_dumps", False)
 
-        if has_dynamic_sampling_minimum_sample_rate(obj.organization, user):
-            data["dynamicSamplingMinimumSampleRate"] = bool(
-                obj.get_option("sentry:dynamic_sampling_minimum_sample_rate")
-            )
-
         return data
 
     def format_options(self, attrs: Mapping[str, Any]) -> dict[str, Any]:

src/sentry/apidocs/examples/project_examples.py

@@ -152,7 +152,6 @@
         "filters:releases": "",
         "filters:error_messages": "",
         "feedback:branding": True,
-        "sentry:dynamic_sampling_minimum_sample_rate": True,
     },
     "digestsMinDelay": 180,
     "digestsMaxDelay": 600,
@@ -260,7 +259,6 @@
         {"id": "boostReplayId", "active": True},
         {"id": "recalibrationRule", "active": True},
     ],
-    "dynamicSamplingMinimumSampleRate": True,
     "eventProcessing": {"symbolicationDegraded": False},
     "symbolSources": "[]",
     "tempestFetchScreenshots": False,

src/sentry/dynamic_sampling/utils.py

@@ -25,18 +25,6 @@ def has_custom_dynamic_sampling(
     )
 
 
-def has_dynamic_sampling_minimum_sample_rate(
-    organization: Organization | None, actor: User | RpcUser | AnonymousUser | None = None
-) -> bool:
-    return (
-        organization is not None
-        and features.has(
-            "organizations:dynamic-sampling-minimum-sample-rate", organization, actor=actor
-        )
-        and has_custom_dynamic_sampling(organization, actor=actor)
-    )
-
-
 def is_project_mode_sampling(organization: Organization | None) -> bool:
     return (
         organization is not None

src/sentry/models/options/project_option.py

@@ -59,7 +59,6 @@
         "sentry:relay_pii_config",
         "sentry:dynamic_sampling",
         "sentry:dynamic_sampling_biases",
-        "sentry:dynamic_sampling_minimum_sample_rate",
         "sentry:target_sample_rate",
         "sentry:tempest_fetch_screenshots",
         "sentry:tempest_fetch_dumps",

src/sentry/projectoptions/defaults.py

@@ -193,9 +193,6 @@
 # Dynamic sampling rate in project-level "manual" configuration mode
 register(key="sentry:target_sample_rate", default=TARGET_SAMPLE_RATE_DEFAULT)
 
-# Dynamic sampling minimum sample rate
-register(key="sentry:dynamic_sampling_minimum_sample_rate", default=False)
-
 # Should tempest fetch screenshots for this project
 register(key="sentry:tempest_fetch_screenshots", default=False)
 

src/sentry/seer/similarity/grouping_records.py

@@ -21,6 +21,7 @@
 logger = logging.getLogger(__name__)
 
 POST_BULK_GROUPING_RECORDS_TIMEOUT = 10000
+DELETE_HASH_METRIC = "grouping.similarity.delete_records_by_hash"
 
 
 class CreateGroupingRecordData(TypedDict):
@@ -125,7 +126,7 @@ def delete_project_grouping_records(
         return False
 
 
-def delete_grouping_records_by_hash(project_id: int, hashes: Sequence[str]) -> bool:
+def call_seer_to_delete_these_hashes(project_id: int, hashes: Sequence[str]) -> bool:
     extra = {"project_id": project_id, "hashes": hashes}
     try:
         body = {"project_id": project_id, "hash_list": hashes}
@@ -142,6 +143,11 @@ def delete_grouping_records_by_hash(project_id: int, hashes: Sequence[str]) -> b
             "seer.delete_grouping_records.hashes.timeout",
             extra=extra,
         )
+        metrics.incr(
+            DELETE_HASH_METRIC,
+            sample_rate=options.get("seer.similarity.metrics_sample_rate"),
+            tags={"success": False, "reason": "ReadTimeoutError"},
+        )
         return False
 
     if response.status >= 200 and response.status < 300:
@@ -150,15 +156,15 @@ def delete_grouping_records_by_hash(project_id: int, hashes: Sequence[str]) -> b
             extra=extra,
         )
         metrics.incr(
-            "grouping.similarity.delete_records_by_hash",
+            DELETE_HASH_METRIC,
             sample_rate=options.get("seer.similarity.metrics_sample_rate"),
             tags={"success": True},
         )
         return True
     else:
         logger.error("seer.delete_grouping_records.hashes.failure", extra=extra)
         metrics.incr(
-            "grouping.similarity.delete_records_by_hash",
+            DELETE_HASH_METRIC,
             sample_rate=options.get("seer.similarity.metrics_sample_rate"),
             tags={"success": False},
         )

src/sentry/tasks/activity.py

@@ -29,7 +29,8 @@ def get_activity_notifiers(project):
     queue="activity.notify",
     silo_mode=SiloMode.REGION,
     taskworker_config=TaskworkerConfig(
-        namespace=notifications_tasks, processing_deadline_duration=60
+        namespace=notifications_tasks,
+        processing_deadline_duration=120,
     ),
 )
 def send_activity_notifications(activity_id: int) -> None:

src/sentry/tasks/collect_project_platforms.py

@@ -31,6 +31,7 @@ def paginate_project_ids(paginate):
     silo_mode=SiloMode.REGION,
     taskworker_config=TaskworkerConfig(
         namespace=issues_tasks,
+        processing_deadline_duration=30,
     ),
 )
 def collect_project_platforms(paginate=1000, **kwargs):

src/sentry/tasks/delete_seer_grouping_records.py

@@ -6,25 +6,24 @@
 from sentry.models.group import Group
 from sentry.models.grouphash import GroupHash
 from sentry.seer.similarity.grouping_records import (
-    delete_grouping_records_by_hash,
+    call_seer_to_delete_these_hashes,
     delete_project_grouping_records,
 )
 from sentry.seer.similarity.utils import ReferrerOptions, killswitch_enabled
 from sentry.silo.base import SiloMode
 from sentry.tasks.base import instrumented_task
 from sentry.taskworker.config import TaskworkerConfig
 from sentry.taskworker.namespaces import seer_tasks
+from sentry.utils import metrics
 from sentry.utils.query import RangeQuerySetWrapper
 
-BATCH_SIZE = 1000
-
 logger = logging.getLogger(__name__)
 
 
 @instrumented_task(
     name="sentry.tasks.delete_seer_grouping_records_by_hash",
     queue="delete_seer_grouping_records_by_hash",
-    max_retries=0,
+    max_retries=0,  # XXX: Why do we not retry?
     silo_mode=SiloMode.REGION,
     soft_time_limit=60 * 15,
     time_limit=60 * (15 + 5),
@@ -42,19 +41,32 @@ def delete_seer_grouping_records_by_hash(
 ) -> None:
     """
     Task to delete seer grouping records by hash list.
-    Calls the seer delete by hash endpoint with batches of hashes of size `BATCH_SIZE`.
+    Calls the seer delete by hash endpoint with batches of hashes of size `batch_size`.
     """
     if killswitch_enabled(project_id, ReferrerOptions.DELETION) or options.get(
         "seer.similarity-embeddings-delete-by-hash-killswitch.enabled"
     ):
         return
 
-    batch_size = options.get("embeddings-grouping.seer.delete-record-batch-size")
+    batch_size = options.get("embeddings-grouping.seer.delete-record-batch-size") or 100
     len_hashes = len(hashes)
-    end_index = min(last_deleted_index + batch_size, len_hashes)
-    delete_grouping_records_by_hash(project_id, hashes[last_deleted_index:end_index])
-    if end_index < len_hashes:
-        delete_seer_grouping_records_by_hash.apply_async(args=[project_id, hashes, end_index])
+    if len_hashes <= batch_size:  # Base case
+        call_seer_to_delete_these_hashes(project_id, hashes)
+    else:
+        if last_deleted_index != 0:
+            # This tracks which tasks are still being scheduled with the whole list of hashes
+            metrics.incr(
+                "grouping.similarity.delete_seer_grouping_records_by_hash.batch_size_exceeded",
+                sample_rate=options.get("seer.similarity.metrics_sample_rate"),
+            )
+
+        # Iterate through hashes in chunks and schedule a task for each chunk
+        # There are tasks passing last_deleted_index, thus, we need to start from that index
+        # Eventually all tasks will pass 0
+        for i in range(last_deleted_index, len_hashes, batch_size):
+            # Slice operations are safe and will not raise IndexError
+            chunked_hashes = hashes[i : i + batch_size]
+            delete_seer_grouping_records_by_hash.apply_async(args=[project_id, chunked_hashes, 0])
 
 
 def call_delete_seer_grouping_records_by_hash(
@@ -71,15 +83,16 @@ def call_delete_seer_grouping_records_by_hash(
         and not options.get("seer.similarity-embeddings-delete-by-hash-killswitch.enabled")
     ):
         group_hashes = []
+        batch_size = options.get("embeddings-grouping.seer.delete-record-batch-size") or 100
 
         for group_hash in RangeQuerySetWrapper(
             GroupHash.objects.filter(project_id=project.id, group__id__in=group_ids),
-            step=BATCH_SIZE,
+            step=batch_size,
         ):
             group_hashes.append(group_hash.hash)
 
-            # Schedule task when we reach BATCH_SIZE
-            if len(group_hashes) >= BATCH_SIZE:
+            # Schedule task when we reach batch_size
+            if len(group_hashes) >= batch_size:
                 delete_seer_grouping_records_by_hash.apply_async(args=[project.id, group_hashes, 0])
                 group_hashes = []