Add support for OpenTelemetry Logs API

LICENSE-3rdparty.csv

@@ -7,6 +7,7 @@ require,@datadog/pprof,Apache license 2.0,Copyright 2019 Google Inc.
 require,@datadog/sketches-js,Apache license 2.0,Copyright 2020 Datadog Inc.
 require,@datadog/wasm-js-rewriter,Apache license 2.0,Copyright 2018 Datadog Inc.
 require,@opentelemetry/api,Apache license 2.0,Copyright OpenTelemetry Authors
+require,@opentelemetry/api-logs,Apache license 2.0,Copyright OpenTelemetry Authors
 require,@opentelemetry/core,Apache license 2.0,Copyright OpenTelemetry Authors
 require,@isaacs/ttlcache,ISC,Copyright (c) 2022-2023 - Isaac Z. Schlueter and Contributors
 require,crypto-randomuuid,MIT,Copyright 2021 Node.js Foundation and contributors

docs/API.md

@@ -379,6 +379,41 @@ The following attributes are available to override Datadog-specific options:
 * `resource.name`: The resource name to be used for this span. The operation name will be used if this is not provided.
 * `span.type`: The span type to be used for this span. Will fallback to `custom` if not provided.
 
+<h3 id="opentelemetry-logs">OpenTelemetry Logs</h3>
+
+dd-trace-js includes experimental support for OpenTelemetry logs, designed as a drop-in replacement for the OpenTelemetry SDK. This support is primarily intended for logging libraries rather than direct user configuration. Enable it by setting `DD_LOGS_OTEL_ENABLED=true` and use the [OpenTelemetry Logs API](https://open-telemetry.github.io/opentelemetry-js/modules/_opentelemetry_api-logs.html) to emit structured log data:
+
+```javascript
+const tracer = require('dd-trace').init()
+const { logs } = require('@opentelemetry/api-logs')
+
+const logger = logs.getLogger('my-service', '1.0.0')
+logger.emit({
+  severityText: 'INFO',
+  severityNumber: 9,
+  body: 'Application started',
+  attributes: { version: '1.0.0' },
+  timestamp: Date.now() * 1000000
+})
+```
+
+#### Supported Configuration
+
+The Datadog SDK supports many of the configurations supported by the OpenTelemetry SDK. The following environment variables are supported:
+
+- `DD_LOGS_OTEL_ENABLED` - Enable OpenTelemetry logs (default: `false`)
+- `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` - OTLP endpoint URL (default: `http://localhost:4318/v1/logs`)
+- `OTEL_EXPORTER_OTLP_LOGS_HEADERS` - Optional headers in JSON format (default: `{}`)
+- `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL` - OTLP protocol for logs (default: `http/protobuf`)
+- `OTEL_EXPORTER_OTLP_PROTOCOL` - OTLP protocol fallback (default: `http/protobuf`)
+- `OTEL_EXPORTER_OTLP_TIMEOUT` - Request timeout in milliseconds (default: `10000`)
+- `OTEL_BSP_SCHEDULE_DELAY` - Batch timeout in milliseconds (default: `5000`)
+- `OTEL_BSP_MAX_EXPORT_BATCH_SIZE` - Maximum logs per batch (default: `512`)
+- `OTEL_BSP_MAX_QUEUE_SIZE` - Maximum queue size (default: `2048`)
+- `OTEL_BSP_EXPORT_TIMEOUT` - Export timeout in milliseconds (default: `30000`)
+
+Logs are exported via OTLP over HTTP. The protocol can be configured using `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL` or `OTEL_EXPORTER_OTLP_PROTOCOL` environment variables. Supported protocols are `http/protobuf` (default) and `http/json`. For complete OTLP exporter configuration options, see the [OpenTelemetry OTLP Exporter documentation](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/).
+
 <h2 id="advanced-configuration">Advanced Configuration</h2>
 
 <h3 id="tracer-settings">Tracer settings</h3>

integration-tests/opentelemetry.spec.js

@@ -133,7 +133,7 @@ describe('opentelemetry', () => {
         OTEL_RESOURCE_ATTRIBUTES: 'foo+bar13baz+qux1',
         DD_TRACE_PROPAGATION_STYLE: 'datadog, tracecontext',
         OTEL_PROPAGATORS: 'datadog, tracecontext',
-        OTEL_LOGS_EXPORTER: 'none',
+        DD_LOGS_OTEL_ENABLED: 'false',
         OTEL_SDK_DISABLED: 'false'
       }
     })
@@ -206,7 +206,7 @@ describe('opentelemetry', () => {
         OTEL_METRICS_EXPORTER: 'foo',
         OTEL_RESOURCE_ATTRIBUTES: 'foo',
         OTEL_PROPAGATORS: 'foo',
-        OTEL_LOGS_EXPORTER: 'foo',
+        DD_LOGS_OTEL_ENABLED: 'foo',
         OTEL_SDK_DISABLED: 'foo'
       }
     })

package.json

@@ -122,6 +122,7 @@
     "@datadog/wasm-js-rewriter": "4.0.1",
     "@isaacs/ttlcache": "^1.4.1",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
+    "@opentelemetry/api-logs": "^0.205.0",
     "@opentelemetry/core": ">=1.14.0 <1.31.0",
     "crypto-randomuuid": "^1.0.0",
     "dc-polyfill": "^0.1.10",

packages/dd-trace/src/config.js

@@ -58,8 +58,8 @@ const otelDdEnvMapping = {
   OTEL_TRACES_EXPORTER: 'DD_TRACE_ENABLED',
   OTEL_METRICS_EXPORTER: 'DD_RUNTIME_METRICS_ENABLED',
   OTEL_RESOURCE_ATTRIBUTES: 'DD_TAGS',
-  OTEL_SDK_DISABLED: 'DD_TRACE_OTEL_ENABLED',
-  OTEL_LOGS_EXPORTER: undefined
+  OTEL_LOGS_EXPORTER: undefined,
+  OTEL_SDK_DISABLED: 'DD_TRACE_OTEL_ENABLED'
 }
 
 const VALID_PROPAGATION_STYLES = new Set(['datadog', 'tracecontext', 'b3', 'b3 single header', 'none'])
@@ -568,6 +568,7 @@ class Config {
       DD_INSTRUMENTATION_TELEMETRY_ENABLED,
       DD_INSTRUMENTATION_CONFIG_ID,
       DD_LOGS_INJECTION,
+      DD_LOGS_OTEL_ENABLED,
       DD_LANGCHAIN_SPAN_CHAR_LIMIT,
       DD_LANGCHAIN_SPAN_PROMPT_COMPLETION_SAMPLE_RATE,
       DD_LLMOBS_AGENTLESS_ENABLED,
@@ -649,7 +650,19 @@ class Config {
       OTEL_RESOURCE_ATTRIBUTES,
       OTEL_SERVICE_NAME,
       OTEL_TRACES_SAMPLER,
-      OTEL_TRACES_SAMPLER_ARG
+      OTEL_TRACES_SAMPLER_ARG,
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT,
+      OTEL_EXPORTER_OTLP_LOGS_HEADERS,
+      OTEL_EXPORTER_OTLP_LOGS_PROTOCOL,
+      OTEL_EXPORTER_OTLP_PROTOCOL,
+      OTEL_EXPORTER_OTLP_TIMEOUT,
+      OTEL_EXPORTERS_OTLP_ENDPOINT,
+      OTEL_EXPORTERS_OTLP_HEADERS,
+      OTEL_EXPORTERS_OTLP_TIMEOUT,
+      OTEL_BSP_SCHEDULE_DELAY,
+      OTEL_BSP_MAX_EXPORT_BATCH_SIZE,
+      OTEL_BSP_MAX_QUEUE_SIZE,
+      OTEL_BSP_EXPORT_TIMEOUT
     } = getEnvironmentVariables()
 
     const tags = {}
@@ -663,6 +676,28 @@ class Config {
     tagger.add(tags, DD_TRACE_TAGS)
     tagger.add(tags, DD_TRACE_GLOBAL_TAGS)
 
+    // OpenTelemetry logs configuration - processed after OTEL_RESOURCE_ATTRIBUTES
+    // Enable logs if DD_LOGS_OTEL_ENABLED is true
+    this._setBoolean(env, 'otelLogsEnabled', isTrue(DD_LOGS_OTEL_ENABLED))
+    // Set OpenTelemetry logs configuration with specific _LOGS_ vars taking precedence over generic _EXPORTERS_ vars
+    this._setString(env, 'otelLogsUrl', OTEL_EXPORTER_OTLP_LOGS_ENDPOINT || OTEL_EXPORTERS_OTLP_ENDPOINT)
+    this._setString(env, 'otelLogsHeaders', OTEL_EXPORTER_OTLP_LOGS_HEADERS || OTEL_EXPORTERS_OTLP_HEADERS)
+    // Handle OTLP protocol with grpc warning
+    const requestedProtocol = OTEL_EXPORTER_OTLP_LOGS_PROTOCOL || OTEL_EXPORTER_OTLP_PROTOCOL
+    if (requestedProtocol === 'grpc') {
+      // eslint-disable-next-line no-console
+      console.warn('OTLP gRPC protocol is not supported for logs. ' +
+        'Defaulting to http/protobuf. gRPC protobuf support may be added in a future release.')
+      this._setString(env, 'otelLogsProtocol', 'http/protobuf')
+    } else {
+      this._setString(env, 'otelLogsProtocol', requestedProtocol || 'http/protobuf')
+    }
+    this._setUnit(env, 'otelLogsTimeout', OTEL_EXPORTER_OTLP_TIMEOUT || OTEL_EXPORTERS_OTLP_TIMEOUT)
+    this._setUnit(env, 'otelLogsBatchTimeout', OTEL_BSP_SCHEDULE_DELAY)
+    this._setUnit(env, 'otelLogsMaxExportBatchSize', OTEL_BSP_MAX_EXPORT_BATCH_SIZE)
+    this._setUnit(env, 'otelLogsMaxQueueSize', OTEL_BSP_MAX_QUEUE_SIZE)
+    this._setUnit(env, 'otelLogsExportTimeoutMillis', OTEL_BSP_EXPORT_TIMEOUT)
+
     this._setBoolean(env, 'apmTracingEnabled', coalesce(
       DD_APM_TRACING_ENABLED,
       DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED && isFalse(DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED)
@@ -1185,6 +1220,7 @@ class Config {
       calc.testManagementAttemptToFixRetries = coalesce(maybeInt(DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES), 20)
       this._setBoolean(calc, 'isImpactedTestsEnabled', !isFalse(DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED))
     }
+
     calc['dogstatsd.hostname'] = this._getHostname()
     this._setBoolean(calc, 'isGitUploadEnabled',
       calc.isIntelligentTestRunnerEnabled && !isFalse(this._isCiVisibilityGitUploadEnabled()))

packages/dd-trace/src/config_defaults.js

@@ -119,6 +119,15 @@ module.exports = {
   isTestManagementEnabled: false,
   isImpactedTestsEnabled: false,
   logInjection: true,
+  otelLogsEnabled: false,
+  otelLogsUrl: 'http://localhost:4318/v1/logs',
+  otelLogsHeaders: {},
+  otelLogsProtocol: 'http/protobuf',
+  otelLogsTimeout: 10_000,
+  otelLogsBatchTimeout: 5000,
+  otelLogsMaxExportBatchSize: 512,
+  otelLogsMaxQueueSize: 2048,
+  otelLogsExportTimeoutMillis: 30_000,
   lookup: undefined,
   inferredProxyServicesEnabled: false,
   memcachedCommandEnabled: false,

packages/dd-trace/src/opentelemetry/logs/batch_log_processor.js

@@ -0,0 +1,138 @@
+'use strict'
+
+/**
+ * @fileoverview BatchLogRecordProcessor implementation for OpenTelemetry logs
+ *
+ * Custom implementation to avoid pulling in the full OpenTelemetry SDK.
+ * Based on OTLP Protocol v1.7.0.
+ */
+
+const log = require('../../log')
+
+/**
+ * BatchLogRecordProcessor processes log records in batches for efficient export.
+ *
+ * This implementation follows the OpenTelemetry JavaScript SDK BatchLogRecordProcessor:
+ * https://open-telemetry.github.io/opentelemetry-js/classes/_opentelemetry_sdk-logs.BatchLogRecordProcessor.html
+ *
+ * @class BatchLogRecordProcessor
+ */
+class BatchLogRecordProcessor {
+  /**
+   * Creates a new BatchLogRecordProcessor instance.
+   *
+   * @param {Array} processors - Array of log processors to process batches
+   * @param {Object} config - Configuration options
+   * @param {number} [config.batchTimeout=5000] - Timeout in milliseconds for batch processing
+   * @param {number} [config.maxExportBatchSize=512] - Maximum number of log records per batch
+   * @param {number} [config.maxQueueSize=2048] - Maximum number of log records in queue
+   * @param {number} [config.exportTimeoutMillis=30000] - Timeout for export operations
+   */
+  constructor (processors, config) {
+    this._processors = processors
+    this._config = config
+    this._isShutdown = false
+    this._batchTimeout = config.batchTimeout || 5000 // 5 seconds default
+    this._maxExportBatchSize = config.maxExportBatchSize || 512
+    this._maxQueueSize = config.maxQueueSize || 2048
+    this._exportTimeoutMillis = config.exportTimeoutMillis || 30_000 // 30 seconds default
+
+    this._logRecords = []
+    this._timer = null
+    this._shutdownPromise = null
+  }
+
+  /**
+   * Processes a single log record.
+   *
+   * @param {Object} logRecord - The log record to process
+   */
+  onEmit (logRecord) {
+    if (this._isShutdown) {
+      return
+    }
+
+    this._logRecords.push(logRecord)
+
+    if (this._logRecords.length >= this._maxExportBatchSize) {
+      this._export()
+    } else if (this._logRecords.length === 1) {
+      this._startTimer()
+    }
+  }
+
+  _startTimer () {
+    if (this._timer) {
+      return
+    }
+
+    this._timer = setTimeout(() => {
+      this._export()
+    }, this._batchTimeout)
+  }
+
+  _export () {
+    if (this._logRecords.length === 0) {
+      return
+    }
+
+    const logRecords = this._logRecords.splice(0, this._maxExportBatchSize)
+    this._clearTimer()
+
+    for (const processor of this._processors) {
+      try {
+        processor.export(logRecords, () => {})
+      } catch (error) {
+        log.error('Error in log processor export:', error)
+      }
+    }
+
+    if (this._logRecords.length > 0) {
+      this._startTimer()
+    }
+  }
+
+  _clearTimer () {
+    if (this._timer) {
+      clearTimeout(this._timer)
+      this._timer = null
+    }
+  }
+
+  forceFlush () {
+    return new Promise((resolve) => {
+      if (this._isShutdown) {
+        resolve()
+        return
+      }
+
+      this._export()
+      resolve()
+    })
+  }
+
+  shutdown () {
+    if (this._isShutdown) {
+      return this._shutdownPromise || Promise.resolve()
+    }
+
+    this._isShutdown = true
+    this._shutdownPromise = new Promise((resolve) => {
+      this._clearTimer()
+
+      this._export()
+
+      const shutdownPromises = this._processors.map(processor => {
+        return typeof processor.shutdown === 'function'
+          ? processor.shutdown()
+          : Promise.resolve()
+      })
+
+      Promise.all(shutdownPromises).then(resolve)
+    })
+
+    return this._shutdownPromise
+  }
+}
+
+module.exports = BatchLogRecordProcessor

packages/dd-trace/src/opentelemetry/logs/common.proto

@@ -0,0 +1,59 @@
+syntax = "proto3";
+
+package opentelemetry.proto.common.v1;
+
+option go_package = "go.opentelemetry.io/collector/pdata/pcommon";
+
+// AnyValue is used to represent any type of attribute value. AnyValue may contain a
+// simple scalar or an arbitrary complex structure including arrays and nested objects.
+// AnyValue is a oneof type and can be one of the following:
+// - string_value: A string value.
+// - bool_value: A boolean value.
+// - int_value: An integer value.
+// - double_value: A double value.
+// - array_value: An array of AnyValue values.
+// - kvlist_value: A key-value list of AnyValue values.
+// - bytes_value: A bytes value.
+message AnyValue {
+  oneof value {
+    string string_value = 1;
+    bool bool_value = 2;
+    int64 int_value = 3;
+    double double_value = 4;
+    ArrayValue array_value = 5;
+    KeyValueList kvlist_value = 6;
+    bytes bytes_value = 7;
+  }
+}
+
+// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message
+// since oneof in AnyValue does not allow repeated fields.
+message ArrayValue {
+  // Array of values. The array may be empty (contain 0 elements).
+  repeated AnyValue values = 1;
+}
+
+// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message
+// since oneof in AnyValue does not allow repeated fields.
+message KeyValueList {
+  // A collection of key/value pairs of key-value pairs. The list may be empty (may
+  // contain 0 elements).
+  repeated KeyValue values = 1;
+}
+
+// KeyValue is a key-value pair that is used to store metadata about the telemetry
+// record.
+message KeyValue {
+  string key = 1;
+  AnyValue value = 2;
+}
+
+// InstrumentationScope is a message representing the instrumentation scope information
+// such as the fully qualified name and version.
+message InstrumentationScope {
+  // An empty instrumentation scope name means the name is unknown.
+  string name = 1;
+  string version = 2;
+  repeated KeyValue attributes = 3;
+  uint32 dropped_attributes_count = 4;
+}

packages/dd-trace/src/opentelemetry/logs/index.js

@@ -0,0 +1,22 @@
+'use strict'
+
+/**
+ * @fileoverview OpenTelemetry Logs Implementation for dd-trace-js
+ *
+ * Custom implementation to avoid pulling in the full OpenTelemetry SDK.
+ * Based on OTLP Protocol v1.7.0.
+ */
+
+const LoggerProvider = require('./logger_provider')
+const Logger = require('./logger')
+const BatchLogRecordProcessor = require('./batch_log_processor')
+const OtlpHttpLogExporter = require('./otlp_http_log_exporter')
+const OtlpTransformer = require('./otlp_transformer')
+
+module.exports = {
+  LoggerProvider,
+  Logger,
+  BatchLogRecordProcessor,
+  OtlpHttpLogExporter,
+  OtlpTransformer
+}