feat(cli): telemetry event deploy (#698)

Extension on #631 to provide deploy telemetry events.

---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

---------

Signed-off-by: github-actions <github-actions@github.com>
Co-authored-by: github-actions <github-actions@github.com>
Co-authored-by: Eli Polonsky <epolon@amazon.com>
Co-authored-by: Eli Polonsky <Eli.polonsky@gmail.com>

Author: 3 people

packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts

@@ -188,6 +188,7 @@ export interface CdkCliOptions extends ShellOptions {
   options?: string[];
   neverRequireApproval?: boolean;
   verbose?: boolean;
+  telemetryFile?: string;
 }
 
 export interface CdkDestroyCliOptions extends CdkCliOptions {
@@ -383,6 +384,7 @@ export class TestFixture extends ShellHelper {
       // use events because bar renders bad in tests
       '--progress', 'events',
       ...(skipStackRename ? stackNames : this.fullStackName(stackNames)),
+      ...(options.telemetryFile ? ['--unstable=telemetry', `--telemetry-file=${options.telemetryFile}`] : []),
     ];
   }
 

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-telemetry.integtest.ts

@@ -0,0 +1,57 @@
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import { integTest, withDefaultFixture } from '../../../lib';
+
+jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
+
+integTest(
+  'cdk deploy with telemetry data',
+  withDefaultFixture(async (fixture) => {
+    const telemetryFile = path.join(fixture.integTestDir, 'telemetry.json');
+
+    // Deploy stack while collecting telemetry
+    await fixture.cdkDeploy('test-1', {
+      telemetryFile,
+    });
+    const json = fs.readJSONSync(telemetryFile);
+    expect(json).toEqual([
+      expect.objectContaining({
+        event: expect.objectContaining({
+          command: expect.objectContaining({
+            path: ['deploy', '$STACKS_1'],
+          }),
+          state: 'SUCCEEDED',
+          eventType: 'SYNTH',
+        }),
+        identifiers: expect.objectContaining({
+          eventId: expect.stringContaining(':1'),
+        }),
+      }),
+      expect.objectContaining({
+        event: expect.objectContaining({
+          command: expect.objectContaining({
+            path: ['deploy', '$STACKS_1'],
+          }),
+          state: 'SUCCEEDED',
+          eventType: 'DEPLOY',
+        }),
+        identifiers: expect.objectContaining({
+          eventId: expect.stringContaining(':2'),
+        }),
+      }),
+      expect.objectContaining({
+        event: expect.objectContaining({
+          command: expect.objectContaining({
+            path: ['deploy', '$STACKS_1'],
+          }),
+          state: 'SUCCEEDED',
+          eventType: 'INVOKE',
+        }),
+        identifiers: expect.objectContaining({
+          eventId: expect.stringContaining(':3'),
+        }),
+      }),
+    ]);
+    fs.unlinkSync(telemetryFile);
+  }),
+);

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/synth/cdk-synth-telemetry.integtest.ts

@@ -114,4 +114,3 @@ integTest(
     fs.unlinkSync(telemetryFile);
   }),
 );
-

packages/aws-cdk/lib/cli/cdk-toolkit.ts

@@ -65,6 +65,9 @@ import {
   validateSnsTopicArn,
 } from '../util';
 import { canCollectTelemetry } from './telemetry/collect-telemetry';
+import { cdkCliErrorName } from './telemetry/error';
+import { CLI_PRIVATE_SPAN } from './telemetry/messages';
+import type { ErrorDetails } from './telemetry/schema';
 
 // Must use a require() otherwise esbuild complains about calling a namespace
 // eslint-disable-next-line @typescript-eslint/no-require-imports,@typescript-eslint/consistent-type-imports
@@ -519,12 +522,15 @@ export class CdkToolkit {
       const stackIndex = stacks.indexOf(stack) + 1;
       await this.ioHost.asIoHelper().defaults.info(`${chalk.bold(stack.displayName)}: deploying... [${stackIndex}/${stackCollection.stackCount}]`);
       const startDeployTime = new Date().getTime();
-
       let tags = options.tags;
       if (!tags || tags.length === 0) {
         tags = tagsForStack(stack);
       }
 
+      // There is already a startDeployTime constant, but that does not work with telemetry.
+      // We should integrate the two in the future
+      const deploySpan = await this.ioHost.asIoHelper().span(CLI_PRIVATE_SPAN.DEPLOY).begin({});
+      let error: ErrorDetails | undefined;
       let elapsedDeployTime = 0;
       try {
         let deployResult: SuccessfulDeployStackResult | undefined;
@@ -638,10 +644,18 @@ export class CdkToolkit {
       } catch (e: any) {
         // It has to be exactly this string because an integration test tests for
         // "bold(stackname) failed: ResourceNotReady: <error>"
-        throw new ToolkitError(
+        const wrappedError = new ToolkitError(
           [`❌  ${chalk.bold(stack.stackName)} failed:`, ...(e.name ? [`${e.name}:`] : []), formatErrorMessage(e)].join(' '),
         );
+
+        error = {
+          name: cdkCliErrorName(wrappedError.name),
+        };
+
+        throw wrappedError;
       } finally {
+        await deploySpan.end({ error });
+
         if (options.cloudWatchLogMonitor) {
           const foundLogGroupsResult = await findCloudWatchLogGroups(this.props.sdkProvider, asIoHelper(this.ioHost, 'deploy'), stack);
           options.cloudWatchLogMonitor.addLogGroups(

packages/aws-cdk/lib/cli/io-host/cli-io-host.ts

@@ -10,7 +10,8 @@ import type { IoHelper, ActivityPrinterProps, IActivityPrinter } from '../../../
 import { asIoHelper, IO, isMessageRelevantForLevel, CurrentActivityPrinter, HistoryActivityPrinter } from '../../../lib/api-private';
 import { StackActivityProgress } from '../../commands/deploy';
 import { FileTelemetrySink } from '../telemetry/file-sink';
-import { CLI_PRIVATE_IO } from '../telemetry/messages';
+import type { EventResult } from '../telemetry/messages';
+import { CLI_PRIVATE_IO, CLI_TELEMETRY_CODES } from '../telemetry/messages';
 import type { EventType } from '../telemetry/schema';
 import { TelemetrySession } from '../telemetry/session';
 import { isCI } from '../util/ci';
@@ -552,12 +553,12 @@ function targetStreamObject(x: TargetStream): NodeJS.WriteStream | undefined {
   }
 }
 
-function isNoticesMessage(msg: IoMessage<unknown>) {
+function isNoticesMessage(msg: IoMessage<unknown>): msg is IoMessage<void> {
   return IO.CDK_TOOLKIT_I0100.is(msg) || IO.CDK_TOOLKIT_W0101.is(msg) || IO.CDK_TOOLKIT_E0101.is(msg) || IO.CDK_TOOLKIT_I0101.is(msg);
 }
 
-function isTelemetryMessage(msg: IoMessage<unknown>) {
-  return CLI_PRIVATE_IO.CDK_CLI_I1001.is(msg) || CLI_PRIVATE_IO.CDK_CLI_I2001.is(msg);
+function isTelemetryMessage(msg: IoMessage<unknown>): msg is IoMessage<EventResult> {
+  return CLI_TELEMETRY_CODES.some((c) => c.is(msg));
 }
 
 function getEventType(msg: IoMessage<unknown>): EventType {
@@ -566,6 +567,8 @@ function getEventType(msg: IoMessage<unknown>): EventType {
       return 'SYNTH';
     case CLI_PRIVATE_IO.CDK_CLI_I2001.code:
       return 'INVOKE';
+    case CLI_PRIVATE_IO.CDK_CLI_I3001.code:
+      return 'DEPLOY';
     default:
       throw new ToolkitError(`Unrecognized Telemetry Message Code: ${msg.code}`);
   }

packages/aws-cdk/lib/cli/telemetry/messages.ts

@@ -34,6 +34,16 @@ export const CLI_PRIVATE_IO = {
     description: 'Command has finished executing',
     interface: 'EventResult',
   }),
+  CDK_CLI_I3000: make.trace<EventStart>({
+    code: 'CDK_CLI_I3000',
+    description: 'Deploy has started',
+    interface: 'EventStart',
+  }),
+  CDK_CLI_I3001: make.trace<EventResult>({
+    code: 'CDK_CLI_I3001',
+    description: 'Deploy has finished',
+    interface: 'EventResult',
+  }),
 };
 
 /**
@@ -50,4 +60,15 @@ export const CLI_PRIVATE_SPAN = {
     start: CLI_PRIVATE_IO.CDK_CLI_I2000,
     end: CLI_PRIVATE_IO.CDK_CLI_I2001,
   },
+  DEPLOY: {
+    name: 'Deploy',
+    start: CLI_PRIVATE_IO.CDK_CLI_I3000,
+    end: CLI_PRIVATE_IO.CDK_CLI_I3001,
+  },
 } satisfies Record<string, SpanDefinition<any, any>>;
+
+export const CLI_TELEMETRY_CODES = [
+  CLI_PRIVATE_IO.CDK_CLI_I1001,
+  CLI_PRIVATE_IO.CDK_CLI_I2001,
+  CLI_PRIVATE_IO.CDK_CLI_I3001,
+];

packages/aws-cdk/lib/cli/telemetry/schema.ts

@@ -24,7 +24,7 @@ interface SessionEvent {
   readonly command: Command;
 }
 
-export type EventType = 'SYNTH' | 'INVOKE';
+export type EventType = 'SYNTH' | 'INVOKE' | 'DEPLOY';
 export type State = 'ABORTED' | 'FAILED' | 'SUCCEEDED';
 interface Event extends SessionEvent {
   readonly state: State;