diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/package.json b/aws-distro-opentelemetry-node-autoinstrumentation/package.json index 2272c054..2e9570f5 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/package.json +++ b/aws-distro-opentelemetry-node-autoinstrumentation/package.json @@ -41,7 +41,7 @@ ], "exclude": [ "src/third-party/**/*.ts", - "src/otlp-aws-span-exporter.ts" + "src/exporter/otlp/aws/common/aws-authenticator.ts" ] }, "bugs": { @@ -98,13 +98,18 @@ }, "dependencies": { "@opentelemetry/api": "1.9.0", - "@opentelemetry/api-events": "0.57.1", "@opentelemetry/auto-configuration-propagators": "0.3.2", "@opentelemetry/auto-instrumentations-node": "0.56.0", + "@opentelemetry/api-events": "0.57.1", + "@opentelemetry/sdk-events": "0.57.1", + "@opentelemetry/sdk-logs": "0.57.1", "@opentelemetry/core": "1.30.1", "@opentelemetry/exporter-metrics-otlp-grpc": "0.57.1", "@opentelemetry/exporter-metrics-otlp-http": "0.57.1", "@opentelemetry/exporter-trace-otlp-proto": "0.57.1", + "@opentelemetry/exporter-logs-otlp-grpc": "0.57.1", + "@opentelemetry/exporter-logs-otlp-http": "0.57.1", + "@opentelemetry/exporter-logs-otlp-proto": "0.57.1", "@opentelemetry/exporter-zipkin": "1.30.1", "@opentelemetry/id-generator-aws-xray": "1.2.3", "@opentelemetry/instrumentation": "0.57.1", @@ -113,8 +118,6 @@ "@opentelemetry/propagator-aws-xray": "1.26.2", "@opentelemetry/resource-detector-aws": "1.12.0", "@opentelemetry/resources": "1.30.1", - "@opentelemetry/sdk-events": "0.57.1", - "@opentelemetry/sdk-logs": "0.57.1", "@opentelemetry/sdk-metrics": "1.30.1", "@opentelemetry/sdk-node": "0.57.1", "@opentelemetry/sdk-trace-base": "1.30.1", diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-opentelemetry-configurator.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-opentelemetry-configurator.ts index a57590ad..6119b660 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-opentelemetry-configurator.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-opentelemetry-configurator.ts @@ -14,6 +14,9 @@ import { import { OTLPTraceExporter as OTLPGrpcTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'; import { OTLPTraceExporter as OTLPHttpTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; import { OTLPTraceExporter as OTLPProtoTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'; +import { OTLPLogExporter as OTLPGrpcLogExporter } from '@opentelemetry/exporter-logs-otlp-grpc'; +import { OTLPLogExporter as OTLPHttpLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; +import { OTLPLogExporter as OTLPProtoLogExporter } from '@opentelemetry/exporter-logs-otlp-proto'; import { ZipkinExporter } from '@opentelemetry/exporter-zipkin'; import { AWSXRayIdGenerator } from '@opentelemetry/id-generator-aws-xray'; import { Instrumentation } from '@opentelemetry/instrumentation'; @@ -50,25 +53,40 @@ import { SpanProcessor, TraceIdRatioBasedSampler, } from '@opentelemetry/sdk-trace-base'; + +import { + BatchLogRecordProcessor, + ConsoleLogRecordExporter, + LogRecordExporter, + LogRecordProcessor, + SimpleLogRecordProcessor, +} from '@opentelemetry/sdk-logs'; import { SEMRESATTRS_TELEMETRY_AUTO_VERSION } from '@opentelemetry/semantic-conventions'; import { AlwaysRecordSampler } from './always-record-sampler'; import { AttributePropagatingSpanProcessorBuilder } from './attribute-propagating-span-processor-builder'; import { AwsBatchUnsampledSpanProcessor } from './aws-batch-unsampled-span-processor'; import { AwsMetricAttributesSpanExporterBuilder } from './aws-metric-attributes-span-exporter-builder'; import { AwsSpanMetricsProcessorBuilder } from './aws-span-metrics-processor-builder'; -import { OTLPAwsSpanExporter } from './otlp-aws-span-exporter'; +import { OTLPAwsSpanExporter } from './exporter/otlp/aws/traces/otlp-aws-span-exporter'; import { OTLPUdpSpanExporter } from './otlp-udp-exporter'; import { AwsXRayRemoteSampler } from './sampler/aws-xray-remote-sampler'; // This file is generated via `npm run compile` import { LIB_VERSION } from './version'; +import { OTLPAwsLogExporter } from './exporter/otlp/aws/logs/otlp-aws-log-exporter'; +import { AwsBatchLogRecordProcessor } from './exporter/otlp/aws/logs/aws-batch-log-record-processor'; -const XRAY_OTLP_ENDPOINT_PATTERN = '^https://xray\\.([a-z0-9-]+)\\.amazonaws\\.com/v1/traces$'; +const AWS_TRACES_OTLP_ENDPOINT_PATTERN = '^https://xray\\.([a-z0-9-]+)\\.amazonaws\\.com/v1/traces$'; +const AWS_LOGS_OTLP_ENDPOINT_PATTERN = '^https://logs\\.([a-z0-9-]+)\\.amazonaws\\.com/v1/logs$'; + +const AWS_OTLP_LOGS_GROUP_HEADER = 'x-aws-log-group'; +const AWS_OTLP_LOGS_STREAM_HEADER = 'x-aws-log-stream'; const APPLICATION_SIGNALS_ENABLED_CONFIG: string = 'OTEL_AWS_APPLICATION_SIGNALS_ENABLED'; const APPLICATION_SIGNALS_EXPORTER_ENDPOINT_CONFIG: string = 'OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT'; const METRIC_EXPORT_INTERVAL_CONFIG: string = 'OTEL_METRIC_EXPORT_INTERVAL'; const DEFAULT_METRIC_EXPORT_INTERVAL_MILLIS: number = 60000; export const AWS_LAMBDA_FUNCTION_NAME_CONFIG: string = 'AWS_LAMBDA_FUNCTION_NAME'; +export const AGENT_OBSERVABILITY_ENABLED = 'AGENT_OBSERVABILITY_ENABLED'; const AWS_XRAY_DAEMON_ADDRESS_CONFIG: string = 'AWS_XRAY_DAEMON_ADDRESS'; const FORMAT_OTEL_SAMPLED_TRACES_BINARY_PREFIX = 'T1S'; const FORMAT_OTEL_UNSAMPLED_TRACES_BINARY_PREFIX = 'T1U'; @@ -95,6 +113,7 @@ export class AwsOpentelemetryConfigurator { private idGenerator: IdGenerator; private sampler: Sampler; private spanProcessors: SpanProcessor[]; + private logRecordProcessors: LogRecordProcessor[]; private propagator: TextMapPropagator; /** @@ -178,6 +197,7 @@ export class AwsOpentelemetryConfigurator { // default SpanProcessors with Span Exporters wrapped inside AwsMetricAttributesSpanExporter const awsSpanProcessorProvider: AwsSpanProcessorProvider = new AwsSpanProcessorProvider(this.resource); this.spanProcessors = awsSpanProcessorProvider.getSpanProcessors(); + this.logRecordProcessors = AwsLoggerProcessorProvider.getlogRecordProcessors(); AwsOpentelemetryConfigurator.customizeSpanProcessors(this.spanProcessors, this.resource); } @@ -206,6 +226,7 @@ export class AwsOpentelemetryConfigurator { // span processors are specified // https://github.com/open-telemetry/opentelemetry-js/issues/3449 spanProcessors: this.spanProcessors, + logRecordProcessors: this.logRecordProcessors, autoDetectResources: false, textMapPropagator: this.propagator, }; @@ -214,12 +235,12 @@ export class AwsOpentelemetryConfigurator { } static isApplicationSignalsEnabled(): boolean { - const applicationSignalsEnabled: string | undefined = process.env[APPLICATION_SIGNALS_ENABLED_CONFIG]; - if (applicationSignalsEnabled === undefined) { + const isApplicationSignalsEnabled: string | undefined = process.env[APPLICATION_SIGNALS_ENABLED_CONFIG]; + if (isApplicationSignalsEnabled === undefined) { return false; } - return applicationSignalsEnabled.toLowerCase() === 'true'; + return isApplicationSignalsEnabled.toLowerCase() === 'true'; } static customizeSpanProcessors(spanProcessors: SpanProcessor[], resource: Resource): void { @@ -384,6 +405,145 @@ export class ApplicationSignalsExporterProvider { }; } +// The OpenTelemetry Authors code +// AWS Distro for OpenTelemetry JavaScript needs to copy and adapt code from the upstream OpenTelemetry project because the original implementation doesn't expose certain critical components +// needed for AWS-specific customizations. Specifically, the private configureLoggerProviderFromEnv() from the OpenTelemetry SDK, is a key function that allows us to configure logs exporters based on environment variables, +// By implementing our own version of these methods, we can extend the functionality to detect AWS service endpoints and automatically switch to AWS-specific, OTLPAwsLogExporter. +// Long term, we want to contribute these changes to upstream. +// +// https://github.com/open-telemetry/opentelemetry-js/blob/main/experimental/packages/opentelemetry-sdk-node/src/sdk.ts#L443 +// +// The upstream OpenTelemetry SDK has changed its API by deprecating `getEnv()` and +// `getEnvWithoutDefaults()` in favor of specific methods like `getStringListFromEnv` +// and `getStringFromEnv`. Since these newer methods aren't available in our current +// supported version, we've also needed to copy them down here. +// +// https://github.com/open-telemetry/opentelemetry-js/blob/main/packages/opentelemetry-core/src/platform/node/environment.ts#L52 +// https://github.com/open-telemetry/opentelemetry-js/blob/main/packages/opentelemetry-core/src/platform/node/environment.ts#L100 +// +// TODO: Remove getStringListFromEnv and getStringFromEnv implementations +// once we upgrade to @opentelemetry/core 2.0.0 or higher, which provides these methods natively. +// +export class AwsLoggerProcessorProvider { + public static getlogRecordProcessors(): LogRecordProcessor[] { + const exporters = AwsLoggerProcessorProvider.configureLogExportersFromEnv(); + + return exporters.map(exporter => { + if (exporter instanceof ConsoleLogRecordExporter) { + return new SimpleLogRecordProcessor(exporter); + } + if (exporter instanceof OTLPAwsLogExporter) { + return new AwsBatchLogRecordProcessor(exporter); + } + + return new BatchLogRecordProcessor(exporter); + }); + } + + static configureLogExportersFromEnv(): LogRecordExporter[] { + const otlpExporterLogsEndpoint = process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT; + const enabledExporters = AwsLoggerProcessorProvider.getStringListFromEnv('OTEL_LOGS_EXPORTER') ?? []; + + if (enabledExporters.length === 0) { + diag.debug('OTEL_LOGS_EXPORTER is empty. Using default otlp exporter.'); + enabledExporters.push('otlp'); + } + + if (enabledExporters.includes('none')) { + diag.info('OTEL_LOGS_EXPORTER contains "none". Logger provider will not be initialized.'); + return []; + } + + const exporters: LogRecordExporter[] = []; + + enabledExporters.forEach(exporter => { + if (exporter === 'otlp') { + const protocol = ( + AwsLoggerProcessorProvider.getStringFromEnv('OTEL_EXPORTER_OTLP_LOGS_PROTOCOL') ?? + AwsLoggerProcessorProvider.getStringFromEnv('OTEL_EXPORTER_OTLP_PROTOCOL') + )?.trim(); + + switch (protocol) { + case 'grpc': + exporters.push(new OTLPGrpcLogExporter()); + break; + case 'http/json': + exporters.push(new OTLPHttpLogExporter()); + break; + case 'http/protobuf': + if ( + otlpExporterLogsEndpoint && + isAwsOtlpEndpoint(otlpExporterLogsEndpoint, 'logs') && + validateLogsHeaders() + ) { + diag.debug('Detected CloudWatch Logs OTLP endpoint. Switching exporter to OTLPAwsLogExporter'); + exporters.push(new OTLPAwsLogExporter(otlpExporterLogsEndpoint)); + } else { + exporters.push(new OTLPProtoLogExporter()); + } + break; + case undefined: + case '': + exporters.push(new OTLPProtoLogExporter()); + break; + default: + diag.warn(`Unsupported OTLP logs protocol: "${protocol}". Using http/protobuf.`); + if ( + otlpExporterLogsEndpoint && + isAwsOtlpEndpoint(otlpExporterLogsEndpoint, 'logs') && + validateLogsHeaders() + ) { + diag.debug('Detected CloudWatch Logs OTLP endpoint. Switching exporter to OTLPAwsLogExporter'); + exporters.push(new OTLPAwsLogExporter(otlpExporterLogsEndpoint)); + } else { + exporters.push(new OTLPProtoLogExporter()); + } + } + } else if (exporter === 'console') { + exporters.push(new ConsoleLogRecordExporter()); + } else { + diag.warn(`Unsupported OTEL_LOGS_EXPORTER value: "${exporter}". Supported values are: otlp, console, none.`); + } + }); + + return exporters; + } + + /** + * Retrieves a list of strings from an environment variable. + * - Uses ',' as the delimiter. + * - Trims leading and trailing whitespace from each entry. + * - Excludes empty entries. + * - Returns `undefined` if the environment variable is empty or contains only whitespace. + * - Returns an empty array if all entries are empty or whitespace. + * + * @param {string} key - The name of the environment variable to retrieve. + * @returns {string[] | undefined} - The list of strings or `undefined`. + */ + private static getStringListFromEnv(key: string): string[] | undefined { + return AwsLoggerProcessorProvider.getStringFromEnv(key) + ?.split(',') + .map(v => v.trim()) + .filter(s => s !== ''); + } + + /** + * Retrieves a string from an environment variable. + * - Returns `undefined` if the environment variable is empty, unset, or contains only whitespace. + * + * @param {string} key - The name of the environment variable to retrieve. + * @returns {string | undefined} - The string value or `undefined`. + */ + private static getStringFromEnv(key: string): string | undefined { + const raw = process.env[key]; + if (raw == null || raw.trim() === '') { + return undefined; + } + return raw; + } +} +// END The OpenTelemetry Authors code + // The OpenTelemetry Authors code // // ADOT JS needs the logic to (1) get the SpanExporters from Env and then (2) wrap the SpanExporters with AwsMetricAttributesSpanExporter @@ -427,7 +587,7 @@ export class AwsSpanProcessorProvider { private resource: Resource; static configureOtlp(): SpanExporter { - const otlp_exporter_traces_endpoint = process.env['OTEL_EXPORTER_OTLP_TRACES_ENDPOINT']; + const otlpExporterTracesEndpoint = process.env['OTEL_EXPORTER_OTLP_TRACES_ENDPOINT']; // eslint-disable-next-line @typescript-eslint/typedef let protocol = this.getOtlpProtocol(); @@ -444,9 +604,9 @@ export class AwsSpanProcessorProvider { case 'http/json': return new OTLPHttpTraceExporter(); case 'http/protobuf': - if (otlp_exporter_traces_endpoint && isXrayOtlpEndpoint(otlp_exporter_traces_endpoint)) { + if (otlpExporterTracesEndpoint && isAwsOtlpEndpoint(otlpExporterTracesEndpoint, 'xray')) { diag.debug('Detected XRay OTLP Traces endpoint. Switching exporter to OtlpAwsSpanExporter'); - return new OTLPAwsSpanExporter(otlp_exporter_traces_endpoint); + return new OTLPAwsSpanExporter(otlpExporterTracesEndpoint); } return new OTLPProtoTraceExporter(); case 'udp': @@ -454,9 +614,9 @@ export class AwsSpanProcessorProvider { return new OTLPUdpSpanExporter(getXrayDaemonEndpoint(), FORMAT_OTEL_SAMPLED_TRACES_BINARY_PREFIX); default: diag.warn(`Unsupported OTLP traces protocol: ${protocol}. Using http/protobuf.`); - if (otlp_exporter_traces_endpoint && isXrayOtlpEndpoint(otlp_exporter_traces_endpoint)) { + if (otlpExporterTracesEndpoint && isAwsOtlpEndpoint(otlpExporterTracesEndpoint, 'xray')) { diag.debug('Detected XRay OTLP Traces endpoint. Switching exporter to OtlpAwsSpanExporter'); - return new OTLPAwsSpanExporter(otlp_exporter_traces_endpoint); + return new OTLPAwsSpanExporter(otlpExporterTracesEndpoint); } return new OTLPProtoTraceExporter(); } @@ -666,8 +826,51 @@ function getXrayDaemonEndpoint() { return process.env[AWS_XRAY_DAEMON_ADDRESS_CONFIG]; } -function isXrayOtlpEndpoint(otlpEndpoint: string | undefined) { - return otlpEndpoint && new RegExp(XRAY_OTLP_ENDPOINT_PATTERN).test(otlpEndpoint.toLowerCase()); +/** + * Determines if the given endpoint is either the AWS OTLP Traces or Logs endpoint. + */ + +function isAwsOtlpEndpoint(otlpEndpoint: string, service: string): boolean { + const pattern = service === 'xray' ? AWS_TRACES_OTLP_ENDPOINT_PATTERN : AWS_LOGS_OTLP_ENDPOINT_PATTERN; + + return new RegExp(pattern).test(otlpEndpoint.toLowerCase()); +} + +/** + * Checks if x-aws-log-group and x-aws-log-stream are present in the headers in order to send logs to + * AWS OTLP Logs endpoint. + */ +function validateLogsHeaders() { + const logsHeaders = process.env['OTEL_EXPORTER_OTLP_LOGS_HEADERS']; + + if (!logsHeaders) { + diag.warn( + 'Improper configuration: Please configure the environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS ' + + 'to include x-aws-log-group and x-aws-log-stream' + ); + return false; + } + + let filteredLogHeadersCount = 0; + + for (const pair of logsHeaders.split(',')) { + if (pair.includes('=')) { + const [key, value] = pair.split('=', 2); + if ((key === AWS_OTLP_LOGS_GROUP_HEADER || key === AWS_OTLP_LOGS_STREAM_HEADER) && value) { + filteredLogHeadersCount += 1; + } + } + } + + if (filteredLogHeadersCount !== 2) { + diag.warn( + 'Improper configuration: Please configure the environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS ' + + 'to have values for x-aws-log-group and x-aws-log-stream' + ); + return false; + } + + return true; } // END The OpenTelemetry Authors code diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/common/aws-authenticator.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/common/aws-authenticator.ts new file mode 100644 index 00000000..8bf6984e --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/common/aws-authenticator.ts @@ -0,0 +1,92 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { diag } from '@opentelemetry/api'; +import { getNodeVersion } from '../../../../utils'; +let SignatureV4: any; +let HttpRequest: any; +let defaultProvider: any; +let Sha256: any; + +let dependenciesLoaded = false; + +if (getNodeVersion() >= 16) { + try { + defaultProvider = require('@aws-sdk/credential-provider-node').defaultProvider; + Sha256 = require('@aws-crypto/sha256-js').Sha256; + SignatureV4 = require('@smithy/signature-v4').SignatureV4; + HttpRequest = require('@smithy/protocol-http').HttpRequest; + dependenciesLoaded = true; + } catch (error) { + diag.error(`Failed to load required AWS dependency for SigV4 Signing: ${error}`); + } +} else { + diag.error('SigV4 signing requires at least Node major version 16'); +} + +export class AwsAuthenticator { + private region: string; + private service: string; + + constructor(region: string, service: string) { + this.region = region; + this.service = service; + } + + public async authenticate(endpoint: string, headers: Record, serializedData: Uint8Array | undefined) { + // Only do SigV4 Signing if the required dependencies are installed. + if (dependenciesLoaded) { + const url = new URL(endpoint); + + if (serializedData === undefined) { + diag.error('Given serialized data is undefined. Not authenticating.'); + return headers; + } + + const cleanedHeaders = this.removeSigV4Headers(headers); + + const request = new HttpRequest({ + method: 'POST', + protocol: 'https', + hostname: url.hostname, + path: url.pathname, + body: serializedData, + headers: { + ...cleanedHeaders, + host: url.hostname, + }, + }); + + try { + const signer = new SignatureV4({ + credentials: defaultProvider(), + region: this.region, + service: this.service, + sha256: Sha256, + }); + + const signedRequest = await signer.sign(request); + + return signedRequest.headers; + } catch (exception) { + diag.debug( + `Failed to sign/authenticate the given exported Span request to OTLP XRay endpoint with error: ${exception}` + ); + } + } + + return headers; + } + + // Cleans up Sigv4 from headers to avoid accidentally copying them to the new headers + private removeSigV4Headers(headers: Record) { + const newHeaders: Record = {}; + const sigV4Headers = ['x-amz-date', 'authorization', 'x-amz-content-sha256', 'x-amz-security-token']; + + for (const key in headers) { + if (!sigV4Headers.includes(key.toLowerCase())) { + newHeaders[key] = headers[key]; + } + } + return newHeaders; + } +} diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/logs/aws-batch-log-record-processor.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/logs/aws-batch-log-record-processor.ts new file mode 100644 index 00000000..97e0d1bf --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/logs/aws-batch-log-record-processor.ts @@ -0,0 +1,155 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { LogRecord, BatchLogRecordProcessor } from '@opentelemetry/sdk-logs'; +import { AnyValue, AnyValueMap } from '@opentelemetry/api-logs'; +import { callWithTimeout } from '@opentelemetry/core'; +import type { BufferConfig } from '@opentelemetry/sdk-logs'; +import { OTLPAwsLogExporter } from './otlp-aws-log-exporter'; + +export const BASE_LOG_BUFFER_BYTE_SIZE: number = 2000; +export const MAX_LOG_REQUEST_BYTE_SIZE: number = 1048576; + +export class AwsBatchLogRecordProcessor extends BatchLogRecordProcessor { + constructor(exporter: OTLPAwsLogExporter, config?: BufferConfig) { + super(exporter, config); + (this as any)._flushOneBatch = () => this._flushOneBatchIntermediary(); + } + + /** + * Custom implementation of BatchLogRecordProcessor that manages log record batching + * with size-based constraints to prevent exceeding AWS request size limits. + * + * This processor still exports all logs up to maxExportBatchSize but rather than doing exactly + * one export promise, we do an array of export Promises where each exported batch will have an additonal constraint: + * + * If the batch to be exported will have a data size of > 1 MB: + * The batch will be split into multiple exports of sub-batches of data size <= 1 MB. + * + * A unique case is if the sub-batch is of data size > 1 MB, then the sub-batch will have exactly 1 log in it. + * + */ + private _flushOneBatchIntermediary(): Promise { + const processor = this as any; + + processor._clearTimer(); + + if (processor._finishedLogRecords.length === 0) { + return Promise.resolve(); + } + + const logsToExport: LogRecord[] = processor._finishedLogRecords.splice(0, processor._maxExportBatchSize); + let batch: LogRecord[] = []; + let batchDataSize = 0; + const exportPromises: Promise[] = []; + + for (let i = 0; i < logsToExport.length; i += 1) { + const logData = logsToExport[i]; + const logSize = AwsBatchLogRecordProcessor.getSizeOfLog(logData); + + if (batch.length > 0 && batchDataSize + logSize > MAX_LOG_REQUEST_BYTE_SIZE) { + // if batchDataSize > MAX_LOG_REQUEST_BYTE_SIZE then batch.length == 1 + if (batchDataSize > MAX_LOG_REQUEST_BYTE_SIZE) { + (processor._exporter as OTLPAwsLogExporter).setGenAIFlag(); + } + + exportPromises.push(callWithTimeout(processor._export(batch), processor._exportTimeoutMillis)); + batchDataSize = 0; + batch = []; + } + + batchDataSize += logSize; + batch.push(logData); + } + + if (batch.length > 0) { + // if batchDataSize > MAX_LOG_REQUEST_BYTE_SIZE then batch.length == 1 + if (batchDataSize > MAX_LOG_REQUEST_BYTE_SIZE) { + (processor._exporter as OTLPAwsLogExporter).setGenAIFlag(); + } + + exportPromises.push(callWithTimeout(processor._export(batch), processor._exportTimeoutMillis)); + } + + return new Promise((resolve, reject) => { + Promise.all(exportPromises) + .then(() => resolve()) + .catch(reject); + }); + } + + /** + * Calculates the estimated byte size of a log record. + * + * @param log - The LogRecord to calculate the size for + * @returns The estimated size in bytes, including a base buffer size plus the size of the log body + */ + private static getSizeOfLog(log: LogRecord): number { + if (!log.body) { + return BASE_LOG_BUFFER_BYTE_SIZE; + } + return BASE_LOG_BUFFER_BYTE_SIZE + AwsBatchLogRecordProcessor.getSizeOfAnyValue(log.body); + } + + /** + * Calculates the size of an AnyValue type. If AnyValue is an instance of a Map or Array, calculation is truncated to one layer. + * + * @param val - The AnyValue to calculate the size for + * @returns The size in bytes + */ + private static getSizeOfAnyValue(val: AnyValue): number { + // Use a stack to prevent excessive recursive calls + const stack: AnyValue[] = [val]; + let size: number = 0; + let depth: number = 0; + + while (stack.length > 0) { + const nextVal = stack.pop(); + + if (!nextVal) { + continue; + } + + if (typeof nextVal === 'string') { + size += nextVal.length; + continue; + } + + if (typeof nextVal === 'boolean') { + size += nextVal ? 4 : 5; // 'true' or 'false' + continue; + } + + if (typeof nextVal === 'number') { + size += nextVal.toString().length; + continue; + } + + if (nextVal instanceof Uint8Array) { + size += nextVal.byteLength; + continue; + } + + if (depth < 1) { + if (Array.isArray(nextVal)) { + for (const item of nextVal) { + stack.push(item); + } + + // By process of elimination, nextVal has to be a Map + } else { + const map = nextVal as AnyValueMap; + + for (const key in map) { + size += key.length; + stack.push(map[key]); + } + } + + depth += 1; + continue; + } + } + + return size; + } +} diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/logs/otlp-aws-log-exporter.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/logs/otlp-aws-log-exporter.ts new file mode 100644 index 00000000..c1fedb57 --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/logs/otlp-aws-log-exporter.ts @@ -0,0 +1,95 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { OTLPLogExporter as OTLPProtoLogExporter } from '@opentelemetry/exporter-logs-otlp-proto'; +import { CompressionAlgorithm, OTLPExporterNodeConfigBase } from '@opentelemetry/otlp-exporter-base'; + +import { ProtobufLogsSerializer } from '@opentelemetry/otlp-transformer'; +import { ReadableLogRecord } from '@opentelemetry/sdk-logs'; +import { AwsAuthenticator } from '../common/aws-authenticator'; +import { ExportResult, ExportResultCode } from '@opentelemetry/core'; + +/** + * This exporter extends the functionality of the OTLPProtoLogExporter to allow spans to be exported + * to the CloudWatch Logs OTLP endpoint https://logs.[AWSRegion].amazonaws.com/v1/logs. Utilizes the aws-sdk + * library to sign and directly inject SigV4 Authentication to the exported request's headers. ... + * + * This only works with version >=16 Node.js environments. + */ + +export const LARGE_LOG_HEADER = 'x-aws-truncatable-fields'; + +export class OTLPAwsLogExporter extends OTLPProtoLogExporter { + private endpoint: string; + private region: string; + private authenticator: AwsAuthenticator; + private genAIFlag: boolean; + + constructor(endpoint: string, config?: OTLPExporterNodeConfigBase) { + let modifiedConfig: OTLPExporterNodeConfigBase = { + url: endpoint, + compression: CompressionAlgorithm.GZIP, + }; + + if (config) { + modifiedConfig = { + ...config, + url: endpoint, + compression: CompressionAlgorithm.GZIP, + }; + } + + super(modifiedConfig); + this.region = endpoint.split('.')[1]; + this.endpoint = endpoint; + this.genAIFlag = false; + this.authenticator = new AwsAuthenticator(this.region, 'logs'); + } + + /** + * Overrides the upstream implementation of export. All behaviors are the same except if the + * endpoint is the CloudWatch Logs OTLP endpoint, we will sign the request with SigV4 in headers before + * sending it to the endpoint. Otherwise, we will skip signing. + */ + + // Upstream already implements a retry mechanism: + // https://github.com/open-telemetry/opentelemetry-js/blob/main/experimental/packages/otlp-exporter-base/src/retrying-transport.ts + + public override async export( + items: ReadableLogRecord[], + resultCallback: (result: ExportResult) => void + ): Promise { + const serializedLogs: Uint8Array | undefined = ProtobufLogsSerializer.serializeRequest(items); + + if (serializedLogs === undefined) { + resultCallback({ + code: ExportResultCode.FAILED, + error: new Error('Nothing to send'), + }); + return; + } + + const headers = this['_delegate']._transport?._transport?._parameters?.headers(); + + if (headers) { + const signedRequest = await this.authenticator.authenticate(this.endpoint, headers, serializedLogs); + + if (this.genAIFlag) { + signedRequest[LARGE_LOG_HEADER] = 'body/content'; + } else { + delete signedRequest[LARGE_LOG_HEADER]; + } + + // See type: https://github.com/open-telemetry/opentelemetry-js/blob/experimental/v0.57.1/experimental/packages/otlp-exporter-base/src/transport/http-transport-types.ts#L31 + const newHeaders: () => Record = () => signedRequest; + this['_delegate']._transport._transport._parameters.headers = newHeaders; + } + + super.export(items, resultCallback); + this.genAIFlag = false; + } + + public setGenAIFlag() { + this.genAIFlag = true; + } +} diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/traces/otlp-aws-span-exporter.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/traces/otlp-aws-span-exporter.ts new file mode 100644 index 00000000..246f12ac --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/exporter/otlp/aws/traces/otlp-aws-span-exporter.ts @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { OTLPTraceExporter as OTLPProtoTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'; +import { OTLPExporterNodeConfigBase } from '@opentelemetry/otlp-exporter-base'; +import { ProtobufTraceSerializer } from '@opentelemetry/otlp-transformer'; +import { ReadableSpan } from '@opentelemetry/sdk-trace-base'; +import { ExportResult, ExportResultCode } from '@opentelemetry/core'; +import { AwsAuthenticator } from '../common/aws-authenticator'; + +/** + * This exporter extends the functionality of the OTLPProtoTraceExporter to allow spans to be exported + * to the XRay OTLP endpoint https://xray.[AWSRegion].amazonaws.com/v1/traces. Utilizes the aws-sdk + * library to sign and directly inject SigV4 Authentication to the exported request's headers. ... + * + * This only works with version >=16 Node.js environments. + */ +export class OTLPAwsSpanExporter extends OTLPProtoTraceExporter { + private endpoint: string; + private region: string; + private authenticator: AwsAuthenticator; + + constructor(endpoint: string, config?: OTLPExporterNodeConfigBase) { + let modifiedConfig: OTLPExporterNodeConfigBase = { + url: endpoint, + }; + + if (config) { + modifiedConfig = { + ...config, + url: endpoint, + }; + } + + super(modifiedConfig); + this.region = endpoint.split('.')[1]; + this.endpoint = endpoint; + this.authenticator = new AwsAuthenticator(this.region, 'xray'); + } + + /** + * Overrides the upstream implementation of export. All behaviors are the same except if the + * endpoint is an XRay OTLP endpoint, we will sign the request with SigV4 in headers before + * sending it to the endpoint. Otherwise, we will skip signing. + */ + public override async export(items: ReadableSpan[], resultCallback: (result: ExportResult) => void): Promise { + const serializedSpans: Uint8Array | undefined = ProtobufTraceSerializer.serializeRequest(items); + + if (serializedSpans === undefined) { + resultCallback({ + code: ExportResultCode.FAILED, + error: new Error('Nothing to send'), + }); + return; + } + + const headers = this['_delegate']._transport?._transport?._parameters?.headers(); + + if (headers) { + const signedRequest = await this.authenticator.authenticate(this.endpoint, headers, serializedSpans); + + // See type: https://github.com/open-telemetry/opentelemetry-js/blob/experimental/v0.57.1/experimental/packages/otlp-exporter-base/src/transport/http-transport-types.ts#L31 + const newHeaders: () => Record = () => signedRequest; + this['_delegate']._transport._transport._parameters.headers = newHeaders; + } + + super.export(items, resultCallback); + } +} diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/register.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/register.ts index bdea7eb9..0a1381e4 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/src/register.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/register.ts @@ -14,6 +14,7 @@ if (process.env.OTEL_TRACES_SAMPLER === 'xray') { } import { diag, DiagConsoleLogger, trace } from '@opentelemetry/api'; +import { logs } from '@opentelemetry/api-logs'; import { getNodeAutoInstrumentations, InstrumentationConfigMap } from '@opentelemetry/auto-instrumentations-node'; import { Instrumentation } from '@opentelemetry/instrumentation'; import * as opentelemetry from '@opentelemetry/sdk-node'; @@ -83,6 +84,9 @@ try { diag.info('Setting TraceProvider for instrumentations at the end of initialization'); for (const instrumentation of instrumentations) { instrumentation.setTracerProvider(trace.getTracerProvider()); + if (instrumentation.setLoggerProvider) { + instrumentation.setLoggerProvider(logs.getLoggerProvider()); + } } diag.debug(`Environment variable OTEL_PROPAGATORS is set to '${process.env.OTEL_PROPAGATORS}'`); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/common/aws-authenticator.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/common/aws-authenticator.test.ts new file mode 100644 index 00000000..7096b498 --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/common/aws-authenticator.test.ts @@ -0,0 +1,171 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, it, beforeEach } from 'mocha'; +import * as proxyquire from 'proxyquire'; +import * as sinon from 'sinon'; +import expect from 'expect'; +import { AwsAuthenticator } from '../../../../../src/exporter/otlp/aws/common/aws-authenticator'; +import { + AUTHORIZATION_HEADER, + AWS_AUTH_PATH, + AWS_HTTP_MODULE, + AWS_OTLP_TRACES_ENDPOINT, + CREDENTIAL_PROVIDER_MODULE, + SHA_256_MODULE, + SIGNATURE_V4_MODULE, + X_AMZ_DATE_HEADER, + X_AMZ_SECURITY_TOKEN_HEADER, +} from './test-utils.test'; +import { getNodeVersion } from '../../../../../src/utils'; + +const mockCredentials = { + accessKeyId: 'test_access_key', + secretAccessKey: 'test_secret_key', + sessionToken: 'test_session_token', +}; + +// Sigv4 is only enabled for node version >= 16 +const version = getNodeVersion(); + +describe('AwsAuthenticator', () => { + let sandbox: sinon.SinonSandbox; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe('should not inject SigV4 Headers if required modules are not available', async () => { + const dependencies = [SIGNATURE_V4_MODULE, CREDENTIAL_PROVIDER_MODULE, SHA_256_MODULE, AWS_HTTP_MODULE]; + + dependencies.forEach(dependency => { + it(`should not sign headers if missing dependency: ${dependency}`, async () => { + Object.keys(require.cache).forEach(key => { + delete require.cache[key]; + }); + + const requireStub = sandbox.stub(require('module'), '_load'); + requireStub.withArgs(dependency).throws(new Error(`Cannot find module '${dependency}'`)); + requireStub.callThrough(); + + const { AwsAuthenticator: MockThrowableModuleAuthenticator } = require(AWS_AUTH_PATH); + + const result = await new MockThrowableModuleAuthenticator('us-east-1', 'xray').authenticate( + AWS_OTLP_TRACES_ENDPOINT, + {}, + new Uint8Array() + ); + + expect(result).not.toHaveProperty(AUTHORIZATION_HEADER); + expect(result).not.toHaveProperty(X_AMZ_DATE_HEADER); + expect(result).not.toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER); + }); + }); + }); + + it('should not inject SigV4 Headers if serialized data is undefined', async () => { + const authenticator = new AwsAuthenticator('us-east-1', 'xray'); + const result = await authenticator.authenticate(AWS_OTLP_TRACES_ENDPOINT, {}, undefined); + + expect(result).not.toHaveProperty(AUTHORIZATION_HEADER); + expect(result).not.toHaveProperty(X_AMZ_DATE_HEADER); + expect(result).not.toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER); + }); + + it('should inject SigV4 Headers', async () => { + const expected = { + [AUTHORIZATION_HEADER]: 'testAuth', + [X_AMZ_DATE_HEADER]: 'testDate', + [X_AMZ_SECURITY_TOKEN_HEADER]: 'testSecurityToken', + }; + + const AwsAuthenticatorWithMock = proxyquire(AWS_AUTH_PATH, { + [CREDENTIAL_PROVIDER_MODULE]: { + defaultProvider: () => Promise.resolve(mockCredentials), + }, + [SIGNATURE_V4_MODULE]: { + SignatureV4: class { + constructor() {} + sign(request: any) { + return Promise.resolve({ + headers: expected, + }); + } + }, + }, + }).AwsAuthenticator; + + const result = await new AwsAuthenticatorWithMock('us-east-1', 'xray').authenticate( + AWS_OTLP_TRACES_ENDPOINT, + { test: 'test' }, + new Uint8Array() + ); + + if (version >= 16) { + expect(result).toHaveProperty(AUTHORIZATION_HEADER); + expect(result).toHaveProperty(X_AMZ_DATE_HEADER); + expect(result).toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER); + + expect(result[AUTHORIZATION_HEADER]).toBe(expected[AUTHORIZATION_HEADER]); + expect(result[X_AMZ_DATE_HEADER]).toBe(expected[X_AMZ_DATE_HEADER]); + expect(result[X_AMZ_SECURITY_TOKEN_HEADER]).toBe(expected[X_AMZ_SECURITY_TOKEN_HEADER]); + } else { + expect(result).not.toHaveProperty(AUTHORIZATION_HEADER); + expect(result).not.toHaveProperty(X_AMZ_DATE_HEADER); + expect(result).not.toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER); + } + }); + + it('should clear SigV4 headers if already present ', async () => { + const notExpected = { + [AUTHORIZATION_HEADER]: 'notExpectedAuth', + [X_AMZ_DATE_HEADER]: 'notExpectedDate', + [X_AMZ_SECURITY_TOKEN_HEADER]: 'notExpectedSecurityToken', + }; + + const expected = { + [AUTHORIZATION_HEADER]: 'testAuth', + [X_AMZ_DATE_HEADER]: 'testDate', + [X_AMZ_SECURITY_TOKEN_HEADER]: 'testSecurityToken', + }; + + const AwsAuthenticatorWithMock = proxyquire(AWS_AUTH_PATH, { + [CREDENTIAL_PROVIDER_MODULE]: { + defaultProvider: () => Promise.resolve(mockCredentials), + }, + [SIGNATURE_V4_MODULE]: { + SignatureV4: class { + constructor() {} + sign(request: any) { + return Promise.resolve({ + headers: expected, + }); + } + }, + }, + }).AwsAuthenticator; + + const result = await new AwsAuthenticatorWithMock('us-east-1', 'xray').authenticate( + AWS_OTLP_TRACES_ENDPOINT, + notExpected, + new Uint8Array() + ); + expect(result).toHaveProperty(AUTHORIZATION_HEADER); + expect(result).toHaveProperty(X_AMZ_DATE_HEADER); + expect(result).toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER); + + if (version >= 16) { + expect(result[AUTHORIZATION_HEADER]).toBe(expected[AUTHORIZATION_HEADER]); + expect(result[X_AMZ_DATE_HEADER]).toBe(expected[X_AMZ_DATE_HEADER]); + expect(result[X_AMZ_SECURITY_TOKEN_HEADER]).toBe(expected[X_AMZ_SECURITY_TOKEN_HEADER]); + } else { + expect(result[AUTHORIZATION_HEADER]).toBe(notExpected[AUTHORIZATION_HEADER]); + expect(result[X_AMZ_DATE_HEADER]).toBe(notExpected[X_AMZ_DATE_HEADER]); + expect(result[X_AMZ_SECURITY_TOKEN_HEADER]).toBe(notExpected[X_AMZ_SECURITY_TOKEN_HEADER]); + } + }); +}); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/common/test-utils.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/common/test-utils.test.ts new file mode 100644 index 00000000..cdf04829 --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/common/test-utils.test.ts @@ -0,0 +1,20 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +export const AWS_AUTH_PATH = '../../../../../src/exporter/otlp/aws/common/aws-authenticator'; +export const AWS_SPAN_EXPORTER_PATH = '../../../../../src/exporter/otlp/aws/traces/otlp-aws-span-exporter'; +export const AWS_LOG_EXPORTER_PATH = '../../../../../src/exporter/otlp/aws/logs/otlp-aws-log-exporter'; + +export const SIGNATURE_V4_MODULE = '@smithy/signature-v4'; +export const CREDENTIAL_PROVIDER_MODULE = '@aws-sdk/credential-provider-node'; +export const SHA_256_MODULE = '@aws-crypto/sha256-js'; +export const AWS_HTTP_MODULE = '@smithy/protocol-http'; + +export const AWS_OTLP_TRACES_ENDPOINT = 'https://xray.us-east-1.amazonaws.com'; +export const AWS_OTLP_TRACES_ENDPOINT_PATH = '/v1/traces'; + +export const AWS_OTLP_LOGS_ENDPOINT = 'https://logs.us-east-1.amazonaws.com'; +export const AWS_OTLP_LOGS_ENDPOINT_PATH = '/v1/logs'; + +export const AUTHORIZATION_HEADER = 'authorization'; +export const X_AMZ_DATE_HEADER = 'x-amz-date'; +export const X_AMZ_SECURITY_TOKEN_HEADER = 'x-amz-security-token'; diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/logs/aws-batch-log-record-processor.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/logs/aws-batch-log-record-processor.test.ts new file mode 100644 index 00000000..2e7427df --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/logs/aws-batch-log-record-processor.test.ts @@ -0,0 +1,164 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import * as sinon from 'sinon'; +import { + MAX_LOG_REQUEST_BYTE_SIZE, + AwsBatchLogRecordProcessor, + BASE_LOG_BUFFER_BYTE_SIZE, +} from '../../../../../src/exporter/otlp/aws/logs/aws-batch-log-record-processor'; +import { LogRecord } from '@opentelemetry/sdk-logs'; +import { AnyValue, SeverityNumber, LogRecord as apiLogRecord } from '@opentelemetry/api-logs'; +import { DEFAULT_ATTRIBUTE_COUNT_LIMIT, ExportResultCode } from '@opentelemetry/core'; +import { LoggerProviderSharedState } from '@opentelemetry/sdk-logs/build/src/internal/LoggerProviderSharedState'; +import expect from 'expect'; +import { IResource } from '@opentelemetry/resources'; + +describe('AwsBatchLogRecordProcessor', () => { + let mockExporter: any; + let processor: AwsBatchLogRecordProcessor; + + beforeEach(() => { + mockExporter = { + export: sinon.stub().callsFake((logs, resultCallback) => { + resultCallback({ code: ExportResultCode.SUCCESS }); + return; + }), + setGenAIFlag: sinon.stub().callsFake(() => { + return; + }), + shutdown: sinon.stub().callsFake(() => { + return; + }), + }; + + processor = new AwsBatchLogRecordProcessor(mockExporter); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('_flushOneBatch', () => { + it('should export single batch if under size limit', async () => { + const logLength = 10; + const testLogs = generateTestLogData(logLength); + + // Add logs to the processor queue + for (const log of testLogs) { + processor.onEmit(log); + } + + await (processor as any)._flushOneBatch(); + + expect((processor as any)._finishedLogRecords.length).toBe(0); + expect(mockExporter.export.callCount).toBe(1); + + const exportedBatch = mockExporter.export.getCalls()[0].args[0]; + + expect(exportedBatch.length).toBe(logLength); + expect(mockExporter.setGenAIFlag.callCount).toBe(0); + }); + + it('should make multiple export calls of batch size 1 to export logs of size > 1 MB', async () => { + const logLength = 10; + const largeLogBody = { + content: 'test'.repeat(MAX_LOG_REQUEST_BYTE_SIZE + 1), + test: 'test', + }; + const testLogs = generateTestLogData(logLength, largeLogBody); + + // Add logs to the processor queue + for (const log of testLogs) { + processor.onEmit(log); + } + + await (processor as any)._flushOneBatch(); + + expect((processor as any)._finishedLogRecords.length).toBe(0); + expect(mockExporter.export.callCount).toBe(logLength); + + const exportCalls = mockExporter.export.getCalls(); + + for (let i = 0; i < exportCalls.length; i += 1) { + const batch = exportCalls[i].args[0]; + expect(batch.length).toBe(1); + } + + expect(mockExporter.setGenAIFlag.callCount).toBe(10); + }); + + it('should correctly batch logs of mixed sizes with appropriate export calls', async () => { + const largeLogBody = { + content: 'test'.repeat(MAX_LOG_REQUEST_BYTE_SIZE + 1), + test: 'test', + }; + const oneTenthSizeLogBody = ['a'.repeat(MAX_LOG_REQUEST_BYTE_SIZE / 10 - BASE_LOG_BUFFER_BYTE_SIZE)]; + + // expect a total of 5 export calls: + + // 1st, 2nd, 3rd batch = export of batch size 1 + // 4th batch = export of batch size 10 + // 5th batch = export of batch size 2 + + const expectedBatchLength: Record = { + 0: 1, // 1st batch should have 1 log + 1: 1, // 2nd batch should have 1 log + 2: 1, // 3rd batch should have 1 log + 3: 10, // 4th batch should have 10 logs + 4: 2, // 5th batch should have 2 logs + }; + const testLogs = generateTestLogData(3, largeLogBody).concat(generateTestLogData(12, oneTenthSizeLogBody)); + + for (const log of testLogs) { + processor.onEmit(log); + } + + await (processor as any)._flushOneBatch(); + + expect((processor as any)._finishedLogRecords.length).toBe(0); + expect(mockExporter.export.callCount).toBe(5); + + const exportCalls = mockExporter.export.getCalls(); + + for (let i = 0; i < exportCalls.length; i += 1) { + const batch = exportCalls[i].args[0]; + expect(batch.length).toBe(expectedBatchLength[i]); + } + + expect(mockExporter.setGenAIFlag.callCount).toBe(3); + }); + }); + + function generateTestLogData(count: number = 1, body?: AnyValue): LogRecord[] { + const logs: LogRecord[] = []; + + for (let i = 0; i < count; i++) { + const sharedState: LoggerProviderSharedState = { + resource: {} as IResource, + forceFlushTimeoutMillis: 10000, + logRecordLimits: { + attributeValueLengthLimit: DEFAULT_ATTRIBUTE_COUNT_LIMIT, + attributeCountLimit: DEFAULT_ATTRIBUTE_COUNT_LIMIT, + }, + loggers: new Map(), + activeProcessor: processor, + registeredLogRecordProcessors: [], + }; + + const logRecord: apiLogRecord = { + timestamp: Date.now(), + severityNumber: SeverityNumber.INFO, + severityText: 'INFO', + body: body ? body : `Test log message ${i}`, + attributes: { 'test.attribute': i }, + }; + + const log = new LogRecord(sharedState, { name: 'test-scope', version: '1.0.0' }, logRecord); + + logs.push(log); + } + + return logs; + } +}); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/logs/otlp-aws-log-exporter.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/logs/otlp-aws-log-exporter.test.ts new file mode 100644 index 00000000..1193ac96 --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/logs/otlp-aws-log-exporter.test.ts @@ -0,0 +1,105 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import expect from 'expect'; +import { + AWS_OTLP_LOGS_ENDPOINT, + AWS_OTLP_LOGS_ENDPOINT_PATH, + AUTHORIZATION_HEADER, + X_AMZ_DATE_HEADER, + X_AMZ_SECURITY_TOKEN_HEADER, + AWS_LOG_EXPORTER_PATH, +} from '../common/test-utils.test'; +import * as sinon from 'sinon'; +import * as proxyquire from 'proxyquire'; +import * as nock from 'nock'; +import { LARGE_LOG_HEADER, OTLPAwsLogExporter } from '../../../../../src/exporter/otlp/aws/logs/otlp-aws-log-exporter'; + +const EXPECTED_AUTH_HEADER = 'AWS4-HMAC-SHA256 Credential=test_key/some_date/us-east-1/logs/aws4_request'; +const EXPECTED_AUTH_X_AMZ_DATE = 'some_date'; +const EXPECTED_AUTH_SECURITY_TOKEN = 'test_token'; + +describe('OTLPAwsLogExporter', () => { + let sandbox: sinon.SinonSandbox; + let exporter: OTLPAwsLogExporter; + let scope: nock.Scope; + let mockModule: any; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + + scope = nock(AWS_OTLP_LOGS_ENDPOINT) + .post(AWS_OTLP_LOGS_ENDPOINT_PATH) + .reply((uri: any, requestBody: any) => { + return [200, '']; + }); + + mockModule = proxyquire(AWS_LOG_EXPORTER_PATH, { + '../common/aws-authenticator': { + AwsAuthenticator: class MockAwsAuthenticator { + constructor() {} + async authenticate(endpoint: string, headers: Record) { + return { + ...headers, + [AUTHORIZATION_HEADER]: EXPECTED_AUTH_HEADER, + [X_AMZ_DATE_HEADER]: EXPECTED_AUTH_X_AMZ_DATE, + [X_AMZ_SECURITY_TOKEN_HEADER]: EXPECTED_AUTH_SECURITY_TOKEN, + }; + } + }, + }, + }); + + exporter = new mockModule.OTLPAwsLogExporter(AWS_OTLP_LOGS_ENDPOINT + AWS_OTLP_LOGS_ENDPOINT_PATH); + }); + + afterEach(() => { + sandbox.restore(); + }); + + it('Should inject SigV4 Headers successfully', done => { + exporter + .export([], () => {}) + .then(() => { + scope.on('request', (req, interceptor, body) => { + const headers = req.headers; + expect(headers).toHaveProperty(AUTHORIZATION_HEADER.toLowerCase()); + expect(headers).toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER.toLowerCase()); + expect(headers).toHaveProperty(X_AMZ_DATE_HEADER.toLowerCase()); + expect(headers).not.toHaveProperty(LARGE_LOG_HEADER); + + expect(headers[AUTHORIZATION_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_HEADER); + expect(headers[X_AMZ_SECURITY_TOKEN_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_SECURITY_TOKEN); + expect(headers[X_AMZ_DATE_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_X_AMZ_DATE); + + expect(headers['content-type']).toBe('application/x-protobuf'); + expect(headers['user-agent']).toMatch(/^OTel-OTLP-Exporter-JavaScript\/\d+\.\d+\.\d+$/); + done(); + }); + }); + }); + + it('Should inject Large Log Headers if Gen AI flag is set to true', done => { + exporter.setGenAIFlag(); + exporter + .export([], () => {}) + .then(() => { + scope.on('request', (req, interceptor, body) => { + const headers = req.headers; + expect(headers).toHaveProperty(AUTHORIZATION_HEADER.toLowerCase()); + expect(headers).toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER.toLowerCase()); + expect(headers).toHaveProperty(X_AMZ_DATE_HEADER.toLowerCase()); + expect(headers).toHaveProperty(LARGE_LOG_HEADER); + + expect(headers[AUTHORIZATION_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_HEADER); + expect(headers[X_AMZ_SECURITY_TOKEN_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_SECURITY_TOKEN); + expect(headers[X_AMZ_DATE_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_X_AMZ_DATE); + + expect(headers['content-type']).toBe('application/x-protobuf'); + expect(headers['user-agent']).toMatch(/^OTel-OTLP-Exporter-JavaScript\/\d+\.\d+\.\d+$/); + expect(headers[LARGE_LOG_HEADER]).toBe('body/content'); + done(); + }); + }); + }); +}); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/traces/otlp-aws-span-exporter.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/traces/otlp-aws-span-exporter.test.ts new file mode 100644 index 00000000..4e968dbc --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/exporter/otlp/aws/traces/otlp-aws-span-exporter.test.ts @@ -0,0 +1,77 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import expect from 'expect'; +import { + AWS_OTLP_TRACES_ENDPOINT, + AUTHORIZATION_HEADER, + X_AMZ_DATE_HEADER, + X_AMZ_SECURITY_TOKEN_HEADER, + AWS_OTLP_TRACES_ENDPOINT_PATH, + AWS_SPAN_EXPORTER_PATH, +} from '../common/test-utils.test'; +import * as sinon from 'sinon'; +import * as proxyquire from 'proxyquire'; +import * as nock from 'nock'; + +const EXPECTED_AUTH_HEADER = 'AWS4-HMAC-SHA256 Credential=test_key/some_date/us-east-1/xray/aws4_request'; +const EXPECTED_AUTH_X_AMZ_DATE = 'some_date'; +const EXPECTED_AUTH_SECURITY_TOKEN = 'test_token'; + +describe('OTLPAwsSpanExporter', () => { + let sandbox: sinon.SinonSandbox; + let scope: nock.Scope; + let mockModule: any; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + + scope = nock(AWS_OTLP_TRACES_ENDPOINT) + .post(AWS_OTLP_TRACES_ENDPOINT_PATH) + .reply((uri: any, requestBody: any) => { + return [200, '']; + }); + + mockModule = proxyquire(AWS_SPAN_EXPORTER_PATH, { + '../common/aws-authenticator': { + AwsAuthenticator: class MockAwsAuthenticator { + constructor() {} + async authenticate(endpoint: string, headers: Record) { + return { + ...headers, + [AUTHORIZATION_HEADER]: EXPECTED_AUTH_HEADER, + [X_AMZ_DATE_HEADER]: EXPECTED_AUTH_X_AMZ_DATE, + [X_AMZ_SECURITY_TOKEN_HEADER]: EXPECTED_AUTH_SECURITY_TOKEN, + }; + } + }, + }, + }); + }); + + afterEach(() => { + sandbox.restore(); + }); + + it('Should inject SigV4 Headers successfully', done => { + const exporter = new mockModule.OTLPAwsSpanExporter(AWS_OTLP_TRACES_ENDPOINT + AWS_OTLP_TRACES_ENDPOINT_PATH); + + exporter + .export([], () => {}) + .then(() => { + scope.on('request', (req, interceptor, body) => { + const headers = req.headers; + expect(headers).toHaveProperty(AUTHORIZATION_HEADER.toLowerCase()); + expect(headers).toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER.toLowerCase()); + expect(headers).toHaveProperty(X_AMZ_DATE_HEADER.toLowerCase()); + + expect(headers[AUTHORIZATION_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_HEADER); + expect(headers[X_AMZ_SECURITY_TOKEN_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_SECURITY_TOKEN); + expect(headers[X_AMZ_DATE_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_X_AMZ_DATE); + + expect(headers['content-type']).toBe('application/x-protobuf'); + expect(headers['user-agent']).toMatch(/^OTel-OTLP-Exporter-JavaScript\/\d+\.\d+\.\d+$/); + done(); + }); + }); + }); +}); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/otlp-aws-span-exporter.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/otlp-aws-span-exporter.test.ts deleted file mode 100644 index 4002293c..00000000 --- a/aws-distro-opentelemetry-node-autoinstrumentation/test/otlp-aws-span-exporter.test.ts +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import expect from 'expect'; -import { OTLPAwsSpanExporter } from '../src/otlp-aws-span-exporter'; -import * as sinon from 'sinon'; -import * as proxyquire from 'proxyquire'; -import * as nock from 'nock'; -import { getNodeVersion } from '../src/utils'; - -const XRAY_OTLP_ENDPOINT = 'https://xray.us-east-1.amazonaws.com'; -const XRAY_OTLP_ENDPOINT_PATH = '/v1/traces'; -const AUTHORIZATION_HEADER = 'Authorization'; -const X_AMZ_DATE_HEADER = 'X-Amz-Date'; -const X_AMZ_SECURITY_TOKEN_HEADER = 'X-Amz-Security-Token'; - -const EXPECTED_AUTH_HEADER = 'AWS4-HMAC-SHA256 Credential=test_key/some_date/us-east-1/xray/aws4_request'; -const EXPECTED_AUTH_X_AMZ_DATE = 'some_date'; -const EXPECTED_AUTH_SECURITY_TOKEN = 'test_token'; - -const nodeVersion = getNodeVersion(); - -// SigV4 exporter requires packages that require Node environments >= 16 -/* istanbul ignore next */ -if (nodeVersion >= 16) { - describe('OTLPAwsSpanExporter', () => { - let sandbox: sinon.SinonSandbox; - let scope: nock.Scope; - let mockModule: any; - - beforeEach(() => { - sandbox = sinon.createSandbox(); - - scope = nock(XRAY_OTLP_ENDPOINT) - .post(XRAY_OTLP_ENDPOINT_PATH) - .reply((uri: any, requestBody: any) => { - return [200, '']; - }); - - mockModule = proxyquire('../src/otlp-aws-span-exporter', { - '@smithy/signature-v4': { - SignatureV4: class MockSignatureV4 { - sign(req: any) { - req.headers = { - ...req.headers, - [AUTHORIZATION_HEADER]: EXPECTED_AUTH_HEADER, - [X_AMZ_DATE_HEADER]: EXPECTED_AUTH_X_AMZ_DATE, - [X_AMZ_SECURITY_TOKEN_HEADER]: EXPECTED_AUTH_SECURITY_TOKEN, - }; - - return req; - } - }, - }, - '@aws-sdk/credential-provider-node': { - defaultProvider: () => async () => { - return { - accessKeyId: 'test_access_key', - secretAccessKey: 'test_secret_key', - }; - }, - }, - }); - }); - - afterEach(() => { - sandbox.restore(); - }); - - it('Should inject SigV4 Headers successfully', done => { - const exporter = new mockModule.OTLPAwsSpanExporter(XRAY_OTLP_ENDPOINT + XRAY_OTLP_ENDPOINT_PATH); - - exporter - .export([], () => {}) - .then(() => { - scope.on('request', (req, interceptor, body) => { - const headers = req.headers; - expect(headers).toHaveProperty(AUTHORIZATION_HEADER.toLowerCase()); - expect(headers).toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER.toLowerCase()); - expect(headers).toHaveProperty(X_AMZ_DATE_HEADER.toLowerCase()); - - expect(headers[AUTHORIZATION_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_HEADER); - expect(headers[X_AMZ_SECURITY_TOKEN_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_SECURITY_TOKEN); - expect(headers[X_AMZ_DATE_HEADER.toLowerCase()]).toBe(EXPECTED_AUTH_X_AMZ_DATE); - - expect(headers['content-type']).toBe('application/x-protobuf'); - expect(headers['user-agent']).toMatch(/^OTel-OTLP-Exporter-JavaScript\/\d+\.\d+\.\d+$/); - done(); - }); - }); - }); - - describe('Should not inject SigV4 headers if dependencies are missing', () => { - const dependencies = [ - '@aws-sdk/credential-provider-node', - '@aws-crypto/sha256-js', - '@smithy/signature-v4', - '@smithy/protocol-http', - ]; - - dependencies.forEach(dependency => { - it(`should not sign headers if missing dependency: ${dependency}`, done => { - const exporter = new OTLPAwsSpanExporter(XRAY_OTLP_ENDPOINT + XRAY_OTLP_ENDPOINT_PATH); - - Object.keys(require.cache).forEach(key => { - delete require.cache[key]; - }); - const requireStub = sandbox.stub(require('module'), '_load'); - requireStub.withArgs(dependency).throws(new Error(`Cannot find module '${dependency}'`)); - requireStub.callThrough(); - - exporter - .export([], () => {}) - .then(() => { - scope.on('request', (req, interceptor, body) => { - const headers = req.headers; - expect(headers).not.toHaveProperty(AUTHORIZATION_HEADER); - expect(headers).not.toHaveProperty(X_AMZ_DATE_HEADER); - expect(headers).not.toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER); - - expect(headers['content-type']).toBe('application/x-protobuf'); - expect(headers['user-agent']).toMatch(/^OTel-OTLP-Exporter-JavaScript\/\d+\.\d+\.\d+$/); - done(); - }); - }); - }); - }); - }); - - it('should not inject SigV4 headers if failure to sign headers', done => { - const stubbedModule = proxyquire('../src/otlp-aws-span-exporter', { - '@smithy/signature-v4': { - SignatureV4: class MockSignatureV4 { - sign() { - throw new Error('signing error'); - } - }, - }, - }); - - const exporter = new stubbedModule.OTLPAwsSpanExporter(XRAY_OTLP_ENDPOINT + XRAY_OTLP_ENDPOINT_PATH); - - exporter - .export([], () => {}) - .then(() => { - scope.on('request', (req, interceptor, body) => { - const headers = req.headers; - expect(headers).not.toHaveProperty(AUTHORIZATION_HEADER); - expect(headers).not.toHaveProperty(X_AMZ_DATE_HEADER); - expect(headers).not.toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER); - - expect(headers['content-type']).toBe('application/x-protobuf'); - expect(headers['user-agent']).toMatch(/^OTel-OTLP-Exporter-JavaScript\/\d+\.\d+\.\d+$/); - done(); - }); - }); - }); - - it('should not inject SigV4 headers if failure to retrieve credentials', done => { - const stubbedModule = proxyquire('../src/otlp-aws-span-exporter', { - '@aws-sdk/credential-provider-node': { - defaultProvider: () => async () => { - throw new Error('credentials error'); - }, - }, - }); - - const exporter = new stubbedModule.OTLPAwsSpanExporter(XRAY_OTLP_ENDPOINT + XRAY_OTLP_ENDPOINT_PATH); - - exporter - .export([], () => {}) - .then(() => { - scope.on('request', (req, interceptor, body) => { - const headers = req.headers; - expect(headers).not.toHaveProperty(AUTHORIZATION_HEADER); - expect(headers).not.toHaveProperty(X_AMZ_DATE_HEADER); - expect(headers).not.toHaveProperty(X_AMZ_SECURITY_TOKEN_HEADER); - - expect(headers['content-type']).toBe('application/x-protobuf'); - expect(headers['user-agent']).toMatch(/^OTel-OTLP-Exporter-JavaScript\/\d+\.\d+\.\d+$/); - done(); - }); - }); - }); - }); -} diff --git a/package-lock.json b/package-lock.json index 9618c1a8..6243662c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,9 @@ "@opentelemetry/auto-configuration-propagators": "0.3.2", "@opentelemetry/auto-instrumentations-node": "0.56.0", "@opentelemetry/core": "1.30.1", + "@opentelemetry/exporter-logs-otlp-grpc": "0.57.1", + "@opentelemetry/exporter-logs-otlp-http": "0.57.1", + "@opentelemetry/exporter-logs-otlp-proto": "0.57.1", "@opentelemetry/exporter-metrics-otlp-grpc": "0.57.1", "@opentelemetry/exporter-metrics-otlp-http": "0.57.1", "@opentelemetry/exporter-trace-otlp-proto": "0.57.1", @@ -4116,7 +4119,6 @@ "version": "0.57.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api-events/-/api-events-0.57.1.tgz", "integrity": "sha512-Th/89vlzvDiqUgb4Bv0l9uBPbsvwMT6IenHCPDbO7cTlnq/Si+1MvuGsDOhZbQeZu398ILPe3G/KraKxdjSTCg==", - "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.3.0", "@opentelemetry/api-logs": "0.57.1" @@ -5376,7 +5378,6 @@ "version": "0.57.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-events/-/sdk-events-0.57.1.tgz", "integrity": "sha512-zi9f/EUCGPp2OktqeW5qYEoWNVOhF23t+bus9VxPmER+xtz+/lw/+1xaE+Co/rKhlK5NJycGhb517P/wuqrODQ==", - "license": "Apache-2.0", "dependencies": { "@opentelemetry/api-events": "0.57.1", "@opentelemetry/api-logs": "0.57.1",