fix(aci): Retry process_delayed_workflows timeouts

Author: kcons

src/sentry/workflow_engine/processors/delayed_workflow.py

@@ -65,6 +65,7 @@
 )
 from sentry.workflow_engine.processors.workflow_fire_history import create_workflow_fire_histories
 from sentry.workflow_engine.tasks.actions import build_trigger_action_task_params, trigger_action
+from sentry.workflow_engine.tasks.utils import retry_timeouts
 from sentry.workflow_engine.types import DataConditionHandler, WorkflowEventData
 from sentry.workflow_engine.utils import log_context
 
@@ -760,6 +761,7 @@ def repr_keys[T, V](d: dict[T, V]) -> dict[str, V]:
     ),
 )
 @retry
+@retry_timeouts
 @log_context.root()
 def process_delayed_workflows(
     project_id: int, batch_key: str | None = None, *args: Any, **kwargs: Any

src/sentry/workflow_engine/tasks/utils.py

@@ -1,3 +1,6 @@
+from functools import wraps
+
+import sentry_sdk
 from google.api_core.exceptions import DeadlineExceeded, RetryError, ServiceUnavailable
 
 from sentry import nodestore
@@ -6,6 +9,8 @@
 from sentry.issues.issue_occurrence import IssueOccurrence
 from sentry.models.environment import Environment
 from sentry.models.group import Group
+from sentry.taskworker.retry import retry_task
+from sentry.taskworker.workerchild import ProcessingDeadlineExceeded
 from sentry.types.activity import ActivityType
 from sentry.utils.retries import ConditionalRetryPolicy, exponential_delay
 from sentry.workflow_engine.types import WorkflowEventData
@@ -82,3 +87,20 @@ def build_workflow_event_data_from_event(
         has_escalated=has_escalated,
         workflow_env=workflow_env,
     )
+
+
+def retry_timeouts(func):
+    """
+    Schedule a task retry if the function raises ProcessingDeadlineExceeded.
+    This exists because the standard retry decorator doesn't allow BaseExceptions.
+    """
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except ProcessingDeadlineExceeded:
+            sentry_sdk.capture_exception(level="info")
+            retry_task()
+
+    return wrapper