fix(aci): Report Snuba exceptions as warnings and retry (#94582)

Snuba errors are expected and should not be treated as errors; when they
occur, we'd like to retry.
We log them as warnings before retry so we still have record of them; if
we used `@retry` for this, it'd report them as high priority and we'd
like to avoid that in this context.

Author: kcons

src/sentry/workflow_engine/processors/delayed_workflow.py

@@ -32,11 +32,12 @@
 from sentry.tasks.post_process import should_retry_fetch
 from sentry.taskworker.config import TaskworkerConfig
 from sentry.taskworker.namespaces import issues_tasks
-from sentry.taskworker.retry import Retry
+from sentry.taskworker.retry import Retry, retry_task
 from sentry.utils import metrics
 from sentry.utils.iterators import chunked
 from sentry.utils.registry import NoRegistrationExistsError
 from sentry.utils.retries import ConditionalRetryPolicy, exponential_delay
+from sentry.utils.snuba import SnubaError
 from sentry.workflow_engine.handlers.condition.event_frequency_query_handlers import (
     BaseEventFrequencyQueryHandler,
     GroupValues,
@@ -817,7 +818,12 @@ def process_delayed_workflows(
         },
     )
 
-    condition_group_results = get_condition_group_results(condition_groups)
+    try:
+        condition_group_results = get_condition_group_results(condition_groups)
+    except SnubaError:
+        # We expect occasional errors, so we report as warning and retry.
+        logger.warning("delayed_workflow.snuba_error", exc_info=True)
+        retry_task()
 
     logger.info(
         "delayed_workflow.condition_group_results",