Feat: Client Reports (#829)

Author: denrase

CHANGELOG.md

@@ -1,9 +1,12 @@
 # Changelog
 
-## Unreleased: 
+## Unreleased
 
+* Feat: Client Reports (#829)
 * Fix: Add missing iOS contexts (#761)
 
+Starting with version `6.6.0` of `sentry`, [Sentry's version >= v21.9.0](https://github.com/getsentry/self-hosted/releases) is required or you have to manually disable sending client reports via the `sendClientReports` option. This only applies to self-hosted Sentry. If you are using [sentry.io](https://sentry.io), no action is needed.
+
 ## 6.5.1
 
 - Update event contexts (#838)

dart/lib/sentry.dart

@@ -14,6 +14,7 @@ export 'src/protocol.dart';
 export 'src/scope.dart';
 export 'src/sentry.dart';
 export 'src/sentry_envelope.dart';
+export 'src/sentry_envelope_item.dart';
 export 'src/sentry_client.dart';
 export 'src/sentry_options.dart';
 // useful for integrations

dart/lib/src/client_reports/client_report.dart

@@ -0,0 +1,30 @@
+import 'package:meta/meta.dart';
+
+import 'discarded_event.dart';
+import '../utils.dart';
+
+@internal
+class ClientReport {
+  ClientReport(this.timestamp, this.discardedEvents);
+
+  final DateTime? timestamp;
+  final List<DiscardedEvent> discardedEvents;
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+
+    if (timestamp != null) {
+      json['timestamp'] = formatDateAsIso8601WithMillisPrecision(timestamp!);
+    }
+
+    final eventsJson = discardedEvents
+        .map((e) => e.toJson())
+        .where((e) => e.isNotEmpty)
+        .toList(growable: false);
+    if (eventsJson.isNotEmpty) {
+      json['discarded_events'] = eventsJson;
+    }
+
+    return json;
+  }
+}

dart/lib/src/client_reports/client_report_recorder.dart

@@ -0,0 +1,54 @@
+import 'package:meta/meta.dart';
+
+import '../sentry_options.dart';
+import 'client_report.dart';
+import 'discarded_event.dart';
+import 'discard_reason.dart';
+import '../transport/data_category.dart';
+
+@internal
+class ClientReportRecorder {
+  ClientReportRecorder(this._clock);
+
+  final ClockProvider _clock;
+  final Map<_QuantityKey, int> _quantities = {};
+
+  void recordLostEvent(
+      final DiscardReason reason, final DataCategory category) {
+    final key = _QuantityKey(reason, category);
+    var current = _quantities[key] ?? 0;
+    _quantities[key] = current + 1;
+  }
+
+  ClientReport? flush() {
+    if (_quantities.isEmpty) {
+      return null;
+    }
+
+    final events = _quantities.keys.map((key) {
+      final quantity = _quantities[key] ?? 0;
+      return DiscardedEvent(key.reason, key.category, quantity);
+    }).toList(growable: false);
+
+    _quantities.clear();
+
+    return ClientReport(_clock(), events);
+  }
+}
+
+class _QuantityKey {
+  _QuantityKey(this.reason, this.category);
+
+  final DiscardReason reason;
+  final DataCategory category;
+
+  @override
+  int get hashCode => Object.hash(reason, category);
+
+  @override
+  bool operator ==(dynamic other) {
+    return other is _QuantityKey &&
+        other.reason == reason &&
+        other.category == category;
+  }
+}

dart/lib/src/client_reports/discard_reason.dart

@@ -0,0 +1,35 @@
+import 'package:meta/meta.dart';
+
+/// A reason that defines why events were lost, see
+/// https://develop.sentry.dev/sdk/client-reports/#envelope-item-payload.
+@internal
+enum DiscardReason {
+  beforeSend,
+  eventProcessor,
+  sampleRate,
+  networkError,
+  queueOverflow,
+  cacheOverflow,
+  rateLimitBackoff,
+}
+
+extension OutcomeExtension on DiscardReason {
+  String toStringValue() {
+    switch (this) {
+      case DiscardReason.beforeSend:
+        return 'before_send';
+      case DiscardReason.eventProcessor:
+        return 'event_processor';
+      case DiscardReason.sampleRate:
+        return 'sample_rate';
+      case DiscardReason.networkError:
+        return 'network_error';
+      case DiscardReason.queueOverflow:
+        return 'queue_overflow';
+      case DiscardReason.cacheOverflow:
+        return 'cache_overflow';
+      case DiscardReason.rateLimitBackoff:
+        return 'ratelimit_backoff';
+    }
+  }
+}

dart/lib/src/client_reports/discarded_event.dart

@@ -0,0 +1,21 @@
+import 'package:meta/meta.dart';
+
+import 'discard_reason.dart';
+import '../transport/data_category.dart';
+
+@internal
+class DiscardedEvent {
+  DiscardedEvent(this.reason, this.category, this.quantity);
+
+  final DiscardReason reason;
+  final DataCategory category;
+  final int quantity;
+
+  Map<String, dynamic> toJson() {
+    return {
+      'reason': reason.toStringValue(),
+      'category': category.toStringValue(),
+      'quantity': quantity,
+    };
+  }
+}

dart/lib/src/client_reports/noop_client_report_recorder.dart

@@ -0,0 +1,19 @@
+import 'package:meta/meta.dart';
+
+import '../transport/data_category.dart';
+import 'client_report.dart';
+import 'client_report_recorder.dart';
+import 'discard_reason.dart';
+
+@internal
+class NoOpClientReportRecorder implements ClientReportRecorder {
+  const NoOpClientReportRecorder();
+
+  @override
+  ClientReport? flush() {
+    return null;
+  }
+
+  @override
+  void recordLostEvent(DiscardReason reason, DataCategory category) {}
+}

dart/lib/src/hub.dart

@@ -2,15 +2,12 @@ import 'dart:async';
 import 'dart:collection';
 
 import 'package:meta/meta.dart';
+import 'transport/data_category.dart';
 
-import 'protocol.dart';
-import 'scope.dart';
-import 'sentry_client.dart';
-import 'sentry_options.dart';
+import '../sentry.dart';
+import 'client_reports/discard_reason.dart';
 import 'sentry_tracer.dart';
 import 'sentry_traces_sampler.dart';
-import 'sentry_user_feedback.dart';
-import 'tracing.dart';
 
 /// Configures the scope through the callback.
 typedef ScopeCallback = void Function(Scope);
@@ -462,14 +459,18 @@ class Hub {
         'Capturing unfinished transaction: ${transaction.eventId}',
       );
     } else {
+      final item = _peek();
+
       if (!transaction.sampled) {
+        _options.recorder.recordLostEvent(
+          DiscardReason.sampleRate,
+          DataCategory.transaction,
+        );
         _options.logger(
           SentryLevel.warning,
           'Transaction ${transaction.eventId} was dropped due to sampling decision.',
         );
       } else {
-        final item = _peek();
-
         try {
           sentryId = await item.client.captureTransaction(
             transaction,

dart/lib/src/sentry_client.dart

@@ -14,6 +14,9 @@ import 'transport/http_transport.dart';
 import 'transport/noop_transport.dart';
 import 'version.dart';
 import 'sentry_envelope.dart';
+import 'client_reports/client_report_recorder.dart';
+import 'client_reports/discard_reason.dart';
+import 'transport/data_category.dart';
 
 /// Default value for [User.ipAddress]. It gets set when an event does not have
 /// a user and IP address. Only applies if [SentryOptions.sendDefaultPii] is set
@@ -34,10 +37,13 @@ class SentryClient {
 
   /// Instantiates a client using [SentryOptions]
   factory SentryClient(SentryOptions options) {
+    if (options.sendClientReports) {
+      options.recorder = ClientReportRecorder(options.clock);
+    }
     if (options.transport is NoOpTransport) {
-      options.transport = HttpTransport(options, RateLimiter(options.clock));
+      final rateLimiter = RateLimiter(options);
+      options.transport = HttpTransport(options, rateLimiter);
     }
-
     return SentryClient._(options);
   }
 
@@ -53,6 +59,7 @@ class SentryClient {
     dynamic hint,
   }) async {
     if (_sampleRate()) {
+      _recordLostEvent(event, DiscardReason.sampleRate);
       _options.logger(
         SentryLevel.debug,
         'Event ${event.eventId.toString()} was dropped due to sampling decision.',
@@ -87,6 +94,7 @@ class SentryClient {
 
     final beforeSend = _options.beforeSend;
     if (beforeSend != null) {
+      final beforeSendEvent = preparedEvent;
       try {
         preparedEvent = await beforeSend(preparedEvent, hint: hint);
       } catch (exception, stackTrace) {
@@ -98,6 +106,7 @@ class SentryClient {
         );
       }
       if (preparedEvent == null) {
+        _recordLostEvent(beforeSendEvent, DiscardReason.beforeSend);
         _options.logger(
           SentryLevel.debug,
           'Event was dropped by BeforeSend callback',
@@ -264,7 +273,7 @@ class SentryClient {
 
   /// Reports the [envelope] to Sentry.io.
   Future<SentryId?> captureEnvelope(SentryEnvelope envelope) {
-    return _options.transport.send(envelope);
+    return _attachClientReportsAndSend(envelope);
   }
 
   /// Reports the [userFeedback] to Sentry.io.
@@ -273,7 +282,7 @@ class SentryClient {
       userFeedback,
       _options.sdk,
     );
-    return _options.transport.send(envelope);
+    return _attachClientReportsAndSend(envelope);
   }
 
   void close() => _options.httpClient.close();
@@ -296,6 +305,7 @@ class SentryClient {
         );
       }
       if (processedEvent == null) {
+        _recordLostEvent(event, DiscardReason.eventProcessor);
         _options.logger(SentryLevel.debug, 'Event was dropped by a processor');
         break;
       }
@@ -309,4 +319,20 @@ class SentryClient {
     }
     return false;
   }
+
+  void _recordLostEvent(SentryEvent event, DiscardReason reason) {
+    DataCategory category;
+    if (event is SentryTransaction) {
+      category = DataCategory.transaction;
+    } else {
+      category = DataCategory.error;
+    }
+    _options.recorder.recordLostEvent(reason, category);
+  }
+
+  Future<SentryId?> _attachClientReportsAndSend(SentryEnvelope envelope) {
+    final clientReport = _options.recorder.flush();
+    envelope.addClientReport(clientReport);
+    return _options.transport.send(envelope);
+  }
 }

dart/lib/src/sentry_envelope.dart

@@ -1,4 +1,5 @@
 import 'dart:convert';
+import 'client_reports/client_report.dart';
 import 'protocol.dart';
 import 'sentry_item_type.dart';
 import 'sentry_options.dart';
@@ -12,7 +13,7 @@ import 'sentry_user_feedback.dart';
 class SentryEnvelope {
   SentryEnvelope(this.header, this.items);
 
-  /// Header descriping envelope content.
+  /// Header describing envelope content.
   final SentryEnvelopeHeader header;
 
   /// All items contained in the envelope.
@@ -74,7 +75,7 @@ class SentryEnvelope {
       if (length < 0) {
         continue;
       }
-      // Olny attachments should be filtered according to
+      // Only attachments should be filtered according to
       // SentryOptions.maxAttachmentSize
       if (item.header.type == SentryItemType.attachment) {
         if (await item.header.length() > options.maxAttachmentSize) {
@@ -88,4 +89,12 @@ class SentryEnvelope {
       }
     }
   }
+
+  /// Add an envelope item containing client report data.
+  void addClientReport(ClientReport? clientReport) {
+    if (clientReport != null) {
+      final envelopeItem = SentryEnvelopeItem.fromClientReport(clientReport);
+      items.add(envelopeItem);
+    }
+  }
 }