Do not send agent API errors to telemetry logs (#6605)

* Introduce a logging marker (EXCLUDE_TELEMETRY) to exclude a log
  message from telemetry even if other criteria is met. This is meant to
  be used for very frequent errors that are better handled by metrics or
  other reporting mechanisms.
* Use it for agent API errors (which may use
  trace_api.responses/trace_api.errors telemetry metrics, eventually).

Author: smola

communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java

@@ -7,6 +7,7 @@
 import datadog.communication.monitor.DDAgentStatsDClientManager;
 import datadog.communication.monitor.Monitoring;
 import datadog.communication.monitor.Recording;
+import datadog.trace.api.telemetry.LogCollector;
 import datadog.trace.util.Strings;
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -376,7 +377,7 @@ public String getVersion() {
   }
 
   private void errorQueryingEndpoint(String endpoint, Throwable t) {
-    log.debug("Error querying {} at {}", endpoint, agentBaseUrl, t);
+    log.debug(LogCollector.EXCLUDE_TELEMETRY, "Error querying {} at {}", endpoint, agentBaseUrl, t);
   }
 
   public String state() {

dd-java-agent/agent-logging/src/main/java/datadog/trace/logging/ddlogger/DDTelemetryLogger.java

@@ -37,6 +37,9 @@ protected void log(LogLevel level, Marker marker, String msg, Throwable t) {
   }
 
   private void telemetryLog(LogLevel level, Marker marker, String msgOrgFormat, Throwable t) {
+    if (marker == LogCollector.EXCLUDE_TELEMETRY) {
+      return;
+    }
     // Report only messages with Throwable or explicitly marked with SEND_TELEMETRY.
     // This might be extended to error level without throwable.
     if (t == null && marker != LogCollector.SEND_TELEMETRY) {

dd-java-agent/agent-logging/src/test/groovy/datadog/trace/logging/ddlogger/DDTelemetryLoggerTest.groovy

@@ -140,4 +140,22 @@ class DDTelemetryLoggerTest extends LogValidatingSpecification {
     call << allLogLevels
     level << allLogLevels
   }
+
+  void 'do not log #call(EXCLUDE_TELEMETRY, msg, throwable) call with level #level'() {
+    given:
+    def format = "msg"
+    def t = new Exception()
+    def logger = createLogger(level)
+
+    when:
+    logger."$call"(LogCollector.EXCLUDE_TELEMETRY, format, t)
+    def collection = LogCollector.get().drain()
+
+    then:
+    collection.isEmpty()
+
+    where:
+    call << allLogLevels
+    level << allLogLevels
+  }
 }

internal-api/src/main/java/datadog/trace/api/telemetry/LogCollector.java

@@ -14,6 +14,7 @@
 
 public class LogCollector {
   public static final Marker SEND_TELEMETRY = MarkerFactory.getMarker("SEND_TELEMETRY");
+  public static final Marker EXCLUDE_TELEMETRY = MarkerFactory.getMarker("EXCLUDE_TELEMETRY");
   private static final int DEFAULT_MAX_CAPACITY = 1024;
   private static final LogCollector INSTANCE = new LogCollector();
   private final Map<RawLogMessage, AtomicInteger> rawLogMessages;