ref(feedback): move userreport ingest and endpoints to feedback module

src/sentry/api/urls.py

@@ -20,7 +20,6 @@
 from sentry.api.endpoints.organization_events_root_cause_analysis import (
     OrganizationEventsRootCauseAnalysisEndpoint,
 )
-from sentry.api.endpoints.organization_feedback_summary import OrganizationFeedbackSummaryEndpoint
 from sentry.api.endpoints.organization_fork import OrganizationForkEndpoint
 from sentry.api.endpoints.organization_insights_tree import OrganizationInsightsTreeEndpoint
 from sentry.api.endpoints.organization_member_invite.details import (
@@ -100,6 +99,11 @@
 from sentry.explore.endpoints.explore_saved_query_starred_order import (
     ExploreSavedQueryStarredOrderEndpoint,
 )
+from sentry.feedback.endpoints.organization_feedback_summary import (
+    OrganizationFeedbackSummaryEndpoint,
+)
+from sentry.feedback.endpoints.organization_user_reports import OrganizationUserReportsEndpoint
+from sentry.feedback.endpoints.project_user_reports import ProjectUserReportsEndpoint
 from sentry.flags.endpoints.hooks import OrganizationFlagsHooksEndpoint
 from sentry.flags.endpoints.logs import (
     OrganizationFlagLogDetailsEndpoint,
@@ -643,7 +647,6 @@
     OrganizationTraceSpansEndpoint,
 )
 from .endpoints.organization_user_details import OrganizationUserDetailsEndpoint
-from .endpoints.organization_user_reports import OrganizationUserReportsEndpoint
 from .endpoints.organization_user_teams import OrganizationUserTeamsEndpoint
 from .endpoints.organization_users import OrganizationUsersEndpoint
 from .endpoints.project_artifact_bundle_file_details import ProjectArtifactBundleFileDetailsEndpoint
@@ -710,7 +713,6 @@
     ProjectTransactionThresholdOverrideEndpoint,
 )
 from .endpoints.project_transfer import ProjectTransferEndpoint
-from .endpoints.project_user_reports import ProjectUserReportsEndpoint
 from .endpoints.project_user_stats import ProjectUserStatsEndpoint
 from .endpoints.project_users import ProjectUsersEndpoint
 from .endpoints.prompts_activity import PromptsActivityEndpoint

src/sentry/api/endpoints/project_user_reports.py → src/sentry/feedback/endpoints/project_user_reports.py

@@ -16,7 +16,7 @@
 from sentry.api.paginator import DateTimePaginator
 from sentry.api.serializers import UserReportWithGroupSerializer, serialize
 from sentry.feedback.lib.utils import FeedbackCreationSource
-from sentry.ingest.userreport import Conflict, save_userreport
+from sentry.feedback.usecases.ingest.userreport import Conflict, save_userreport
 from sentry.models.environment import Environment
 from sentry.models.userreport import UserReport
 from sentry.utils.dates import epoch

src/sentry/feedback/usecases/save_feedback_event.py → src/sentry/feedback/usecases/ingest/save_event_feedback.py

@@ -4,21 +4,24 @@
 from typing import Any
 
 from sentry.feedback.lib.utils import FeedbackCreationSource
-from sentry.feedback.usecases.create_feedback import create_feedback_issue
-from sentry.ingest.userreport import Conflict, save_userreport
+from sentry.feedback.usecases.ingest.create_feedback import create_feedback_issue
+from sentry.feedback.usecases.ingest.userreport import Conflict, save_userreport
 from sentry.models.environment import Environment
 from sentry.models.project import Project
 from sentry.utils import metrics
 
 logger = logging.getLogger(__name__)
 
 
-def save_feedback_event(event_data: Mapping[str, Any], project_id: int):
-    """Saves a feedback from a feedback event envelope.
+def save_event_feedback(event_data: Mapping[str, Any], project_id: int):
+    """Saves feedback given data in an event format. This function should only
+    be called by the new feedback consumer's ingest strategy, to process
+    feedback envelopes (feedback v2). It is currently instrumented as a task in
+    sentry.tasks.store.
 
-    If the save is successful and the `associated_event_id` field is present, this will
-    also save a UserReport in Postgres. This is to ensure the feedback can be queried by
-    group_id, which is hard to associate in clickhouse.
+    If the save is successful and the `associated_event_id` field is present,
+    this will also save a UserReport in Postgres (shim to v1). This is to allow
+    queries by the group_id relation, which we don't have in clickhouse.
     """
     if not isinstance(event_data, dict):
         event_data = dict(event_data)
@@ -32,6 +35,9 @@ def save_feedback_event(event_data: Mapping[str, Any], project_id: int):
 
     try:
         # Shim to UserReport
+        # TODO: this logic should be extracted to a shim_to_userreport function
+        # which returns a report dict. After that this function can be removed
+        # and the store task can directly call feedback ingest functions.
         feedback_context = fixed_event_data["contexts"]["feedback"]
         associated_event_id = feedback_context.get("associated_event_id")
 

src/sentry/feedback/usecases/shim_to_feedback.py → src/sentry/feedback/usecases/ingest/shim_to_feedback.py

@@ -8,7 +8,7 @@
 from sentry.eventstore.models import Event, GroupEvent
 from sentry.feedback.lib.types import UserReportDict
 from sentry.feedback.lib.utils import FeedbackCreationSource, is_in_feedback_denylist
-from sentry.feedback.usecases.create_feedback import create_feedback_issue
+from sentry.feedback.usecases.ingest.create_feedback import create_feedback_issue
 from sentry.models.project import Project
 from sentry.utils import metrics
 from sentry.utils.outcomes import Outcome, track_outcome

src/sentry/feedback/usecases/ingest/userreport.py

@@ -0,0 +1,265 @@
+from __future__ import annotations
+
+import logging
+import random
+import uuid
+from datetime import datetime, timedelta
+
+from django.core.exceptions import PermissionDenied
+from django.db import IntegrityError, router
+from django.utils import timezone
+
+from sentry import eventstore, options
+from sentry.api.exceptions import BadRequest
+from sentry.constants import DataCategory
+from sentry.eventstore.models import Event, GroupEvent
+from sentry.feedback.lib.types import UserReportDict
+from sentry.feedback.lib.utils import (
+    UNREAL_FEEDBACK_UNATTENDED_MESSAGE,
+    FeedbackCreationSource,
+    is_in_feedback_denylist,
+)
+from sentry.feedback.usecases.ingest.shim_to_feedback import shim_to_feedback
+from sentry.models.project import Project
+from sentry.models.userreport import UserReport
+from sentry.signals import user_feedback_received
+from sentry.utils import metrics
+from sentry.utils.db import atomic_transaction
+from sentry.utils.outcomes import Outcome, track_outcome
+
+logger = logging.getLogger(__name__)
+
+
+class Conflict(Exception):
+    pass
+
+
+def save_userreport(
+    project: Project,
+    report: UserReportDict,
+    source: FeedbackCreationSource,
+    start_time: datetime | None = None,
+) -> UserReport | None:
+    with metrics.timer("userreport.create_user_report", tags={"referrer": source.value}):
+        if start_time is None:
+            start_time = timezone.now()
+
+        if is_in_feedback_denylist(project.organization):
+            metrics.incr(
+                "user_report.create_user_report.filtered",
+                tags={"reason": "org.denylist", "referrer": source.value},
+            )
+            track_outcome(
+                org_id=project.organization_id,
+                project_id=project.id,
+                key_id=None,
+                outcome=Outcome.RATE_LIMITED,
+                reason="feedback_denylist",
+                timestamp=start_time,
+                event_id=None,  # Note report["event_id"] is id of the associated event, not the report itself.
+                category=DataCategory.USER_REPORT_V2,
+                quantity=1,
+            )
+            if (
+                source == FeedbackCreationSource.USER_REPORT_DJANGO_ENDPOINT
+                or source == FeedbackCreationSource.CRASH_REPORT_EMBED_FORM
+            ):
+                raise PermissionDenied()
+            return None
+
+        should_filter, metrics_reason, display_reason = validate_user_report(
+            report, project.id, source=source
+        )
+        if should_filter:
+            metrics.incr(
+                "user_report.create_user_report.filtered",
+                tags={"reason": metrics_reason, "referrer": source.value},
+            )
+            track_outcome(
+                org_id=project.organization_id,
+                project_id=project.id,
+                key_id=None,
+                outcome=Outcome.INVALID,
+                reason=display_reason,
+                timestamp=start_time,
+                event_id=None,  # Note report["event_id"] is id of the associated event, not the report itself.
+                category=DataCategory.USER_REPORT_V2,
+                quantity=1,
+            )
+            if (
+                source == FeedbackCreationSource.USER_REPORT_DJANGO_ENDPOINT
+                or source == FeedbackCreationSource.CRASH_REPORT_EMBED_FORM
+            ):
+                raise BadRequest(display_reason)
+            return None
+
+        # XXX(dcramer): enforce case insensitivity by coercing this to a lowercase string
+        report["event_id"] = report["event_id"].lower()
+        report["project_id"] = project.id
+
+        # Use the associated event to validate and update the report.
+        event: Event | GroupEvent | None = eventstore.backend.get_event_by_id(
+            project.id, report["event_id"]
+        )
+
+        if event:
+            # if the event is more than 30 minutes old, we don't allow updates
+            # as it might be abusive
+            if event.datetime < start_time - timedelta(minutes=30):
+                metrics.incr(
+                    "user_report.create_user_report.filtered",
+                    tags={"reason": "event_too_old", "referrer": source.value},
+                )
+                track_outcome(
+                    org_id=project.organization_id,
+                    project_id=project.id,
+                    key_id=None,
+                    outcome=Outcome.INVALID,
+                    reason="Associated event is too old",
+                    timestamp=start_time,
+                    event_id=None,
+                    category=DataCategory.USER_REPORT_V2,
+                    quantity=1,
+                )
+                raise Conflict("Feedback for this event cannot be modified.")
+
+            report["environment_id"] = event.get_environment().id
+            if event.group_id:
+                report["group_id"] = event.group_id
+
+        # Save the report.
+        try:
+            with atomic_transaction(using=router.db_for_write(UserReport)):
+                report_instance = UserReport.objects.create(**report)
+
+        except IntegrityError:
+            # There was a duplicate, so just overwrite the existing
+            # row with the new one. The only way this ever happens is
+            # if someone is messing around with the API, or doing
+            # something wrong with the SDK, but this behavior is
+            # more reasonable than just hard erroring and is more
+            # expected.
+            existing_report = UserReport.objects.get(
+                project_id=report["project_id"], event_id=report["event_id"]
+            )
+
+            # if the existing report was submitted more than 5 minutes ago, we dont
+            # allow updates as it might be abusive (replay attacks)
+            if existing_report.date_added < timezone.now() - timedelta(minutes=5):
+                metrics.incr(
+                    "user_report.create_user_report.filtered",
+                    tags={"reason": "duplicate_report", "referrer": source.value},
+                )
+                track_outcome(
+                    org_id=project.organization_id,
+                    project_id=project.id,
+                    key_id=None,
+                    outcome=Outcome.INVALID,
+                    reason="Duplicate report",
+                    timestamp=start_time,
+                    event_id=None,
+                    category=DataCategory.USER_REPORT_V2,
+                    quantity=1,
+                )
+                raise Conflict("Feedback for this event cannot be modified.")
+
+            existing_report.update(
+                name=report.get("name", ""),
+                email=report.get("email", ""),
+                comments=report["comments"],
+            )
+            report_instance = existing_report
+
+            metrics.incr(
+                "user_report.create_user_report.overwrite_duplicate",
+                tags={"referrer": source.value},
+            )
+
+        else:
+            if report_instance.group_id:
+                report_instance.notify()
+
+        # Additionally processing if save is successful.
+        user_feedback_received.send_robust(project=project, sender=save_userreport)
+
+        metrics.incr(
+            "user_report.create_user_report.saved",
+            tags={"has_event": bool(event), "referrer": source.value},
+        )
+        if event and source.value in FeedbackCreationSource.old_feedback_category_values():
+            shim_to_feedback(report, event, project, source)
+            # XXX(aliu): the update_user_reports task will still try to shim the report after a period, unless group_id or environment is set.
+
+        return report_instance
+
+
+def validate_user_report(
+    report: UserReportDict,
+    project_id: int,
+    source: FeedbackCreationSource = FeedbackCreationSource.USER_REPORT_ENVELOPE,
+) -> tuple[bool, str | None, str | None]:
+    """
+    Validates required fields, field lengths, and garbage messages. Also checks that event_id is a valid UUID. Does not raise errors.
+
+    Reformatting: strips whitespace from comments and dashes from event_id.
+
+    Returns a tuple of (should_filter, metrics_reason, display_reason). XXX: ensure metrics and outcome reasons have bounded cardinality.
+    """
+    if "comments" not in report:
+        return True, "missing_comments", "Missing comments"  # type: ignore[unreachable]
+    if "event_id" not in report:
+        return True, "missing_event_id", "Missing event_id"  # type: ignore[unreachable]
+
+    report["comments"] = report["comments"].strip()
+
+    name, email, comments = (
+        report.get("name", ""),
+        report.get("email", ""),
+        report["comments"],
+    )
+
+    if options.get("feedback.filter_garbage_messages"):  # Message-based filter kill-switch.
+        if not comments:
+            return True, "empty", "Empty Feedback Messsage"
+
+        if comments == UNREAL_FEEDBACK_UNATTENDED_MESSAGE:
+            return True, "unreal.unattended", "Sent in Unreal Unattended Mode"
+
+    max_comment_length = UserReport._meta.get_field("comments").max_length
+    if max_comment_length and len(comments) > max_comment_length:
+        metrics.distribution(
+            "feedback.large_message",
+            len(comments),
+            tags={
+                "entrypoint": "save_userreport",
+                "referrer": source.value,
+            },
+        )
+        if random.random() < 0.1:
+            logger.info(
+                "Feedback message exceeds max size.",
+                extra={
+                    "project_id": project_id,
+                    "entrypoint": "save_userreport",
+                    "referrer": source.value,
+                    "length": len(comments),
+                    "feedback_message": comments[:100],
+                },
+            )
+        return True, "too_large.message", "Message Too Large"
+
+    max_name_length = UserReport._meta.get_field("name").max_length
+    if max_name_length and len(name) > max_name_length:
+        return True, "too_large.name", "Name Too Large"
+
+    max_email_length = UserReport._meta.get_field("email").max_length
+    if max_email_length and len(email) > max_email_length:
+        return True, "too_large.email", "Email Too Large"
+
+    try:
+        # Validates UUID and strips dashes.
+        report["event_id"] = uuid.UUID(report["event_id"].lower()).hex
+    except ValueError:
+        return True, "invalid_event_id", "Invalid Event ID"
+
+    return False, None, None

src/sentry/ingest/consumer/processors.py

@@ -15,8 +15,8 @@
 from sentry.event_manager import EventManager, save_attachment
 from sentry.eventstore.processing import event_processing_store, transaction_processing_store
 from sentry.feedback.lib.utils import FeedbackCreationSource, is_in_feedback_denylist
+from sentry.feedback.usecases.ingest.userreport import Conflict, save_userreport
 from sentry.ingest.types import ConsumerType
-from sentry.ingest.userreport import Conflict, save_userreport
 from sentry.killswitches import killswitch_matches_context
 from sentry.models.organization import Organization
 from sentry.models.project import Project