AppSignals Functionality - add Application Signals Configuration into ADOT

aws-distro-opentelemetry-node-autoinstrumentation/package.json

@@ -30,7 +30,7 @@
     "prewatch": "npm run precompile",
     "prepublishOnly": "npm run compile",
     "tdd": "yarn test -- --watch-extensions ts --watch",
-    "test": "nyc ts-mocha -p tsconfig.json 'test/**/*.ts'",
+    "test": "nyc ts-mocha --timeout 10000 -p tsconfig.json 'test/**/*.ts'",
     "watch": "tsc -w"
   },
   "bugs": {
@@ -46,15 +46,28 @@
     "sinon": "15.2.0",
     "ts-mocha": "10.0.0",
     "typescript": "4.4.4",
-    "expect": "29.2.0"
+    "expect": "29.2.0",
+    "nock": "13.2.1"
   },
   "dependencies": {
     "@opentelemetry/api": "1.9.0",
     "@opentelemetry/auto-instrumentations-node": "0.48.0",
+    "@opentelemetry/auto-configuration-propagators": "0.2.0",
     "@opentelemetry/exporter-metrics-otlp-grpc": "0.52.1",
     "@opentelemetry/exporter-metrics-otlp-http": "0.52.1",
+    "@opentelemetry/instrumentation": "0.52.1",
     "@opentelemetry/id-generator-aws-xray": "1.2.2",
-    "@opentelemetry/propagator-aws-xray": "1.25.1"
+    "@opentelemetry/propagator-aws-xray": "1.25.1",
+    "@opentelemetry/core": "1.25.1",
+    "@opentelemetry/sdk-trace-base": "1.25.1",
+    "@opentelemetry/semantic-conventions": "1.25.1",
+    "@opentelemetry/resources": "1.25.1",
+    "@opentelemetry/resource-detector-aws": "1.5.2",
+    "@opentelemetry/exporter-trace-otlp-proto": "0.52.1",
+    "@opentelemetry/exporter-zipkin": "1.25.1",
+    "@opentelemetry/sdk-metrics": "1.25.1",
+    "@opentelemetry/sdk-node": "0.52.1",
+    "@opentelemetry/instrumentation-aws-sdk": "0.43.1"
   },
   "overrides": {
     "@opentelemetry/auto-instrumentations-node": {

aws-distro-opentelemetry-node-autoinstrumentation/src/aws-span-processing-util.ts

@@ -115,6 +115,14 @@ export class AwsSpanProcessingUtil {
       return false;
     }
 
+    // TODO:
+    // Telemetry improvements:
+    // - return false for dns instrumentation client spans
+    //   (suppress metrics for spans with name: `dns.lookup`)
+    // - return false for mongoose instrumentation client spans in
+    //   favor of lower level mongodb instrumentation client spans
+    //   (suppress metrics for spans attribute 'db.system': 'mongoose')
+
     return (
       SpanKind.CLIENT === span.kind ||
       SpanKind.PRODUCER === span.kind ||

aws-distro-opentelemetry-node-autoinstrumentation/src/register.ts

@@ -1,5 +1,6 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
+// Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
 
 import { DiagConsoleLogger, diag } from '@opentelemetry/api';
 import * as opentelemetry from '@opentelemetry/sdk-node';
@@ -23,13 +24,21 @@ Also sets default OTEL_PROPAGATORS to ensure good compatibility with X-Ray and A
 This file may also be used to apply patches to upstream instrumentation - usually these are stopgap measures until we can contribute
 long-term changes to upstream.
 */
-
-if (!process.env.OTEL_EXPORTER_OTLP_PROTOCOL) {
-  process.env.OTEL_EXPORTER_OTLP_PROTOCOL = 'http/protobuf';
-}
-if (!process.env.OTEL_PROPAGATORS) {
-  process.env.OTEL_PROPAGATORS = 'xray,tracecontext,b3,b3multi';
+export function setAwsDefaultEnvironmentVariables(): void {
+  if (!process.env.OTEL_EXPORTER_OTLP_PROTOCOL) {
+    process.env.OTEL_EXPORTER_OTLP_PROTOCOL = 'http/protobuf';
+  }
+  if (!process.env.OTEL_PROPAGATORS) {
+    process.env.OTEL_PROPAGATORS = 'xray,tracecontext,b3,b3multi';
+  }
+  // Disable `@opentelemetry/instrumentation-fs` instrumentation by default
+  // This auto-instrumentation for the `fs` module would otherwise generate many low-value spans.
+  // https://github.com/open-telemetry/opentelemetry-js-contrib/issues/1344#issuecomment-1618993178
+  if (!process.env.OTEL_NODE_DISABLED_INSTRUMENTATIONS) {
+    process.env.OTEL_NODE_DISABLED_INSTRUMENTATIONS = 'fs';
+  }
 }
+setAwsDefaultEnvironmentVariables();
 
 const configurator: AwsOpentelemetryConfigurator = new AwsOpentelemetryConfigurator();
 const configuration: Partial<opentelemetry.NodeSDKConfiguration> = configurator.configure();
@@ -43,6 +52,8 @@ const sdk: opentelemetry.NodeSDK = new opentelemetry.NodeSDK(configuration);
 try {
   sdk.start();
   diag.info('AWS Distro of OpenTelemetry automatic instrumentation started successfully');
+  diag.debug(`Environment variable OTEL_PROPAGATORS is set to '${process.env.OTEL_PROPAGATORS}'`);
+  diag.debug(`Environment variable OTEL_EXPORTER_OTLP_PROTOCOL is set to '${process.env.OTEL_EXPORTER_OTLP_PROTOCOL}'`);
 } catch (error) {
   diag.error(
     'Error initializing AWS Distro of OpenTelemetry SDK. Your application is not instrumented and will not produce telemetry',
@@ -56,3 +67,5 @@ process.on('SIGTERM', () => {
     .then(() => diag.debug('AWS Distro of OpenTelemetry SDK terminated'))
     .catch(error => diag.error('Error terminating AWS Distro of OpenTelemetry SDK', error));
 });
+
+// END The OpenTelemetry Authors code

aws-distro-opentelemetry-node-autoinstrumentation/test/aws-opentelemetry-configurator.test.ts

@@ -0,0 +1,205 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { Span, TraceFlags, Tracer } from '@opentelemetry/api';
+import { OTLPMetricExporter as OTLPGrpcOTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc';
+import { OTLPMetricExporter as OTLPHttpOTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
+import { Resource } from '@opentelemetry/resources';
+import { PushMetricExporter } from '@opentelemetry/sdk-metrics';
+import { ReadableSpan, SpanProcessor } from '@opentelemetry/sdk-trace-base';
+import {
+  AlwaysOnSampler,
+  NodeTracerProvider,
+  ParentBasedSampler,
+  Sampler,
+  SpanExporter,
+} from '@opentelemetry/sdk-trace-node';
+import * as assert from 'assert';
+import expect from 'expect';
+import * as sinon from 'sinon';
+import { AlwaysRecordSampler } from '../src/always-record-sampler';
+import { AttributePropagatingSpanProcessor } from '../src/attribute-propagating-span-processor';
+import { AwsMetricAttributesSpanExporter } from '../src/aws-metric-attributes-span-exporter';
+import {
+  ApplicationSignalsExporterProvider,
+  AwsOpentelemetryConfigurator,
+  AwsSpanProcessorProvider,
+  customBuildSamplerFromEnv,
+} from '../src/aws-opentelemetry-configurator';
+import { AwsSpanMetricsProcessor } from '../src/aws-span-metrics-processor';
+import { setAwsDefaultEnvironmentVariables } from '../src/register';
+
+// Tests AwsOpenTelemetryConfigurator after running Environment Variable setup in register.ts
+describe('AwsOpenTelemetryConfiguratorTest', () => {
+  let awsOtelConfigurator: AwsOpentelemetryConfigurator;
+
+  // setUpClass
+  before(() => {
+    // Run environment setup in register.ts, then validate expected env values.
+    setAwsDefaultEnvironmentVariables();
+    validateConfiguratorEnviron();
+
+    // Overwrite exporter configs to keep tests clean, set sampler configs for tests
+    process.env.OTEL_TRACES_EXPORTER = 'none';
+    process.env.OTEL_METRICS_EXPORTER = 'none';
+    process.env.OTEL_LOGS_EXPORTER = 'none';
+    process.env.OTEL_TRACES_SAMPLER = 'traceidratio';
+    process.env.OTEL_TRACES_SAMPLER_ARG = '0.01';
+
+    // Create configurator
+    awsOtelConfigurator = new AwsOpentelemetryConfigurator();
+  });
+
+  // The probability of this passing once without correct IDs is low, 20 times is inconceivable.
+  it('ProvideGenerateXrayIdsTest', () => {
+    const tracerProvider: NodeTracerProvider = new NodeTracerProvider(awsOtelConfigurator.configure());
+    tracerProvider.addSpanProcessor(
+      AttributePropagatingSpanProcessor.create((span: ReadableSpan) => '', 'spanNameKey', ['testKey1', 'testKey2'])
+    );
+    for (let _: number = 0; _ < 20; _++) {
+      const tracer: Tracer = tracerProvider.getTracer('test');
+      const startTimeSec: number = Math.floor(new Date().getTime() / 1000.0);
+      const span: Span = tracer.startSpan('test');
+      const traceId: string = span.spanContext().traceId;
+      const traceId4ByteHex: string = traceId.substring(0, 8);
+      const traceId4ByteNumber: number = Number(`0x${traceId4ByteHex}`);
+      expect(traceId4ByteNumber).toBeGreaterThanOrEqual(startTimeSec);
+    }
+  });
+
+  // Sanity check that the trace ID ratio sampler works fine with the x-ray generator.
+  it('TraceIdRatioSamplerTest', () => {
+    process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED = 'True';
+    const tracerProvider: NodeTracerProvider = new NodeTracerProvider(awsOtelConfigurator.configure());
+    delete process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED;
+
+    tracerProvider.addSpanProcessor(
+      AttributePropagatingSpanProcessor.create((span: ReadableSpan) => '', 'spanNameKey', ['testKey1', 'testKey2'])
+    );
+    for (let _: number = 0; _ < 20; _++) {
+      const numSpans: number = 100000;
+      let numSampled: number = 0;
+      const tracer: Tracer = tracerProvider.getTracer('test');
+      for (let __: number = 0; __ < numSpans; __++) {
+        const span: Span = tracer.startSpan('test');
+        if (span.spanContext().traceFlags & TraceFlags.SAMPLED) {
+          numSampled += 1;
+        }
+        span.end();
+      }
+      // Configured for 1%, confirm there are at most 5% to account for randomness and reduce test flakiness.
+      expect(0.05).toBeGreaterThan(numSampled / numSpans);
+    }
+  });
+
+  it('ImportDefaultSamplerWhenEnvVarIsNotSetTest', () => {
+    delete process.env.OTEL_TRACES_SAMPLER;
+    const defaultSampler: Sampler = customBuildSamplerFromEnv(Resource.empty());
+
+    expect(defaultSampler).not.toBeUndefined();
+    expect(defaultSampler.toString()).toEqual(new ParentBasedSampler({ root: new AlwaysOnSampler() }).toString());
+  });
+
+  it('IsApplicationSignalsEnabledTest', () => {
+    process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED = 'True';
+    expect(AwsOpentelemetryConfigurator.isApplicationSignalsEnabled()).toBeTruthy();
+    delete process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED;
+
+    process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED = 'False';
+    expect(AwsOpentelemetryConfigurator.isApplicationSignalsEnabled()).toBeFalsy();
+    delete process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED;
+    expect(AwsOpentelemetryConfigurator.isApplicationSignalsEnabled()).toBeFalsy();
+  });
+
+  it('CustomizeSamplerTest', () => {
+    const mockSampler: Sampler = sinon.createStubInstance(AlwaysOnSampler);
+    let customizedSampler: Sampler = AwsOpentelemetryConfigurator.customizeSampler(mockSampler);
+    expect(mockSampler).toEqual(customizedSampler);
+
+    process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED = 'True';
+    customizedSampler = AwsOpentelemetryConfigurator.customizeSampler(mockSampler);
+    expect(mockSampler).not.toEqual(customizedSampler);
+    expect(customizedSampler).toBeInstanceOf(AlwaysRecordSampler);
+    expect(mockSampler).toEqual((customizedSampler as any).rootSampler);
+    delete process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED;
+  });
+
+  it('CustomizeExporterTest', () => {
+    const mockExporter: SpanExporter = sinon.createStubInstance(AwsMetricAttributesSpanExporter);
+    let customizedExporter: SpanExporter = AwsSpanProcessorProvider.customizeSpanExporter(
+      mockExporter,
+      Resource.empty()
+    );
+    expect(mockExporter).toEqual(customizedExporter);
+
+    process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED = 'True';
+    customizedExporter = AwsSpanProcessorProvider.customizeSpanExporter(mockExporter, Resource.empty());
+    expect(mockExporter).not.toEqual(customizedExporter);
+    expect(customizedExporter).toBeInstanceOf(AwsMetricAttributesSpanExporter);
+    expect(mockExporter).toEqual((customizedExporter as any).delegate);
+    delete process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED;
+  });
+
+  it('CustomizeSpanProcessorsTest', () => {
+    delete process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED;
+    const spanProcessors: SpanProcessor[] = [];
+    AwsOpentelemetryConfigurator.customizeSpanProcessors(spanProcessors, Resource.empty());
+    expect(spanProcessors.length).toEqual(0);
+
+    process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED = 'True';
+    AwsOpentelemetryConfigurator.customizeSpanProcessors(spanProcessors, Resource.empty());
+    expect(spanProcessors.length).toEqual(2);
+    const firstProcessor: SpanProcessor = spanProcessors[0];
+    expect(firstProcessor).toBeInstanceOf(AttributePropagatingSpanProcessor);
+    const secondProcessor: SpanProcessor = spanProcessors[1];
+    expect(secondProcessor).toBeInstanceOf(AwsSpanMetricsProcessor);
+    delete process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED;
+
+    try {
+      process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED = 'True';
+      process.env.OTEL_METRIC_EXPORT_INTERVAL = undefined;
+      AwsOpentelemetryConfigurator.customizeSpanProcessors(spanProcessors, Resource.empty());
+      process.env.OTEL_METRIC_EXPORT_INTERVAL = '123abc';
+      AwsOpentelemetryConfigurator.customizeSpanProcessors(spanProcessors, Resource.empty());
+      process.env.OTEL_METRIC_EXPORT_INTERVAL = '!@#$%^&*()';
+      AwsOpentelemetryConfigurator.customizeSpanProcessors(spanProcessors, Resource.empty());
+      process.env.OTEL_METRIC_EXPORT_INTERVAL = '123';
+      AwsOpentelemetryConfigurator.customizeSpanProcessors(spanProcessors, Resource.empty());
+    } catch (e: any) {
+      assert.fail(`AwsOpentelemetryConfigurator.customizeSpanProcessors() has incorrectly thrown error: ${e}`);
+    } finally {
+      delete process.env.OTEL_AWS_APPLICATION_SIGNALS_ENABLED;
+    }
+  });
+
+  it('ApplicationSignalsExporterProviderTest', () => {
+    // Check default protocol - HTTP, as specified by aws-distro-opentelemetry-node-autoinstrumentation's register.ts.
+    let exporter: PushMetricExporter = ApplicationSignalsExporterProvider.Instance.createExporter();
+    expect(exporter).toBeInstanceOf(OTLPHttpOTLPMetricExporter);
+    expect('http://localhost:4316/v1/metrics').toEqual((exporter as any)._otlpExporter.url);
+
+    // Overwrite protocol to gRPC.
+    process.env.OTEL_EXPORTER_OTLP_PROTOCOL = 'grpc';
+    exporter = ApplicationSignalsExporterProvider.Instance.createExporter();
+    expect(exporter).toBeInstanceOf(OTLPGrpcOTLPMetricExporter);
+    expect('localhost:4315').toEqual((exporter as any)._otlpExporter.url);
+
+    // Overwrite protocol back to HTTP.
+    process.env.OTEL_EXPORTER_OTLP_PROTOCOL = 'http/protobuf';
+    exporter = ApplicationSignalsExporterProvider.Instance.createExporter();
+    expect(exporter).toBeInstanceOf(OTLPHttpOTLPMetricExporter);
+    expect('http://localhost:4316/v1/metrics').toEqual((exporter as any)._otlpExporter.url);
+  });
+
+  function validateConfiguratorEnviron() {
+    // Set by register.ts
+    expect('http/protobuf').toEqual(process.env.OTEL_EXPORTER_OTLP_PROTOCOL);
+    expect('xray,tracecontext,b3,b3multi').toEqual(process.env.OTEL_PROPAGATORS);
+
+    // Not set
+    expect(undefined).toEqual(process.env.OTEL_TRACES_SAMPLER);
+    expect(undefined).toEqual(process.env.OTEL_TRACES_SAMPLER_ARG);
+    expect(undefined).toEqual(process.env.OTEL_TRACES_EXPORTER);
+    expect(undefined).toEqual(process.env.OTEL_METRICS_EXPORTER);
+  }
+});

aws-distro-opentelemetry-node-autoinstrumentation/test/register.test.ts

@@ -0,0 +1,83 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
+
+import * as assert from 'assert';
+import { spawnSync, SpawnSyncReturns } from 'child_process';
+
+// The OpenTelemetry Authors code
+// Extend register.test.ts functionality to also test exported span with Application Signals enabled
+describe('Register', function () {
+  it('can load auto instrumentation from command line', () => {
+    const proc: SpawnSyncReturns<Buffer> = spawnSync(
+      process.execPath,
+      ['--require', '../build/src/register.js', './third-party/otel/test-app/app.js'],
+      {
+        cwd: __dirname,
+        timeout: 10000,
+        killSignal: 'SIGKILL', // SIGTERM is not sufficient to terminate some hangs
+        env: Object.assign({}, process.env, {
+          OTEL_NODE_RESOURCE_DETECTORS: 'none',
+          OTEL_TRACES_EXPORTER: 'console',
+          // nx (used by lerna run) defaults `FORCE_COLOR=true`, which in
+          // node v18.17.0, v20.3.0 and later results in ANSI color escapes
+          // in the ConsoleSpanExporter output that is checked below.
+          FORCE_COLOR: '0',
+
+          OTEL_LOG_LEVEL: 'ALL',
+          OTEL_TRACES_SAMPLER: 'always_on',
+          OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: 'http://localhost:4316/v1/traces',
+          OTEL_RESOURCE_ATTRIBUTES: 'service.name=test-adot-sdk-ec2-service-name',
+          OTEL_AWS_APPLICATION_SIGNALS_ENABLED: 'true',
+          OTEL_NODE_DISABLED_INSTRUMENTATIONS: 'fs',
+        }),
+      }
+    );
+    assert.ifError(proc.error);
+    assert.equal(proc.status, 0, `proc.status (${proc.status})`);
+    assert.equal(proc.signal, null, `proc.signal (${proc.signal})`);
+
+    assert.ok(proc.stdout.includes('AWS Distro of OpenTelemetry automatic instrumentation started successfully'));
+    assert.ok(proc.stdout.includes("Environment variable OTEL_EXPORTER_OTLP_PROTOCOL is set to 'http/protobuf'"));
+    assert.ok(proc.stdout.includes("Environment variable OTEL_PROPAGATORS is set to 'xray,tracecontext,b3,b3multi'"));
+
+    // Check a span has been generated for the GET request done in app.js
+    assert.ok(proc.stdout.includes("name: 'GET'"), 'console span output in stdout - validate Span Name');
+    assert.ok(
+      proc.stdout.includes("'service.name': 'test-adot-sdk-ec2-service-name'"),
+      'console span output in stdout - validate service.name'
+    );
+
+    // eslint-disable-next-line @typescript-eslint/typedef
+    const packageJson = require('./../../package.json');
+    const DISTRO_VERSION: string = packageJson.version;
+    assert.ok(
+      proc.stdout.includes(`'telemetry.auto.version': '${DISTRO_VERSION}-aws'`),
+      'console span output in stdout - validate telemetry.auto.version'
+    );
+    assert.ok(
+      proc.stdout.includes("'aws.is.local.root': true"),
+      'console span output in stdout - validate aws.is.local.root'
+    );
+    assert.ok(
+      proc.stdout.includes("'aws.local.operation': 'InternalOperation'"),
+      'console span output in stdout - validate aws.local.operation'
+    );
+    assert.ok(
+      proc.stdout.includes("'aws.local.service': 'test-adot-sdk-ec2-service-name'"),
+      'console span output in stdout - validate aws.local.service'
+    );
+    assert.ok(
+      proc.stdout.includes("'aws.remote.service': 'example.com:80'"),
+      'console span output in stdout - validate aws.remote.service'
+    );
+    assert.ok(
+      proc.stdout.includes("'aws.remote.operation': 'GET /'"),
+      'console span output in stdout - validate aws.remote.operation'
+    );
+    assert.ok(
+      proc.stdout.includes("'aws.span.kind': 'LOCAL_ROOT'"),
+      'console span output in stdout - validate aws.span.kind'
+    );
+  });
+});