feat: Implement entrypoint for debug replayer (#848)

Will be used in the VS Code debugger extension

Author: bergundy

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "temporal.replayerEntrypoint": "packages/test/src/debug-replayer.ts"
+}

docs/debug-replay.mermaid

@@ -0,0 +1,19 @@
+sequenceDiagram
+    participant R as Runner (e.g. VS Code)
+    participant MT as Node.js Main Thread
+    participant WF as Workflow Context
+    participant WT as Worker Thread
+
+    R->>MT: Launch
+    MT->>R: Get history
+    MT->>MT: Replay workflow
+    loop
+        MT->>WF: Activate workflow
+        WF->>MT: Update runner with current eventId (blocking call)
+        MT->>WT: Update runner (block using atomics)
+        WT->>R: Set current event ID
+        alt has breakpoint?
+            R->>R: ⏸ Pause execution
+        end
+        R->>WT: Respond OK
+    end

packages/test/src/debug-replayer.ts

@@ -0,0 +1,5 @@
+import { startDebugReplayer } from '@temporalio/worker';
+
+startDebugReplayer({
+  workflowsPath: require.resolve('./workflows'),
+});

packages/test/src/workflows/http.ts

@@ -1,4 +1,3 @@
-// @@@SNIPSTART typescript-schedule-activity-workflow
 import { proxyActivities } from '@temporalio/workflow';
 import type * as activities from '../activities';
 
@@ -9,4 +8,3 @@ const { httpGet } = proxyActivities<typeof activities>({
 export async function http(): Promise<string> {
   return await httpGet('https://temporal.io');
 }
-// @@@SNIPEND

packages/worker/src/debug-replayer/client.ts

@@ -0,0 +1,86 @@
+import http from 'node:http';
+import pkg from '../pkg';
+
+interface ClientOptions {
+  baseUrl: string;
+}
+
+/**
+ * "High level" HTTP client, used to avoid adding more dependencies to the worker package.
+ *
+ * DO NOT use this in a real application, it's meant to only be used for calling a "runner" (e.g. VS Code debugger
+ * extension).
+ */
+export class Client {
+  constructor(public readonly options: ClientOptions) {}
+
+  async post(url: string, options: http.RequestOptions, body: Buffer): Promise<http.IncomingMessage> {
+    const request = http.request(`${this.options.baseUrl}/${url}`, {
+      ...options,
+      method: 'POST',
+      headers: {
+        'Temporal-Client-Name': 'temporal-typescript',
+        'Temporal-Client-Version': pkg.version,
+        'Content-Length': body.length,
+        ...options?.headers,
+      },
+    });
+    if (body) {
+      await new Promise<void>((resolve, reject) => {
+        request.once('error', reject);
+        request.write(body, (err) => {
+          request.off('error', reject);
+          if (err) {
+            reject();
+          } else {
+            resolve();
+          }
+        });
+        request.end();
+      });
+    }
+    const response = await new Promise<http.IncomingMessage>((resolve, reject) => {
+      request.once('error', reject);
+      request.once('response', resolve);
+    });
+    if (response.statusCode !== 200) {
+      throw new Error(`Bad response code from VS Code: ${response.statusCode}`);
+    }
+    return response;
+  }
+
+  async get(url: string, options?: http.RequestOptions): Promise<http.IncomingMessage> {
+    const request = http.get(`${this.options.baseUrl}/${url}`, {
+      ...options,
+      headers: {
+        'Temporal-Client-Name': 'temporal-typescript',
+        'Temporal-Client-Version': pkg.version,
+        ...options?.headers,
+      },
+    });
+    const response = await new Promise<http.IncomingMessage>((resolve, reject) => {
+      request.once('error', reject);
+      request.once('response', resolve);
+    });
+    if (response.statusCode !== 200) {
+      throw new Error(`Bad response code from VS Code: ${response.statusCode}`);
+    }
+    return response;
+  }
+
+  static async readAll(response: http.IncomingMessage): Promise<Buffer> {
+    const chunks = Array<Buffer>();
+    for await (const chunk of response) {
+      chunks.push(chunk);
+    }
+    return Buffer.concat(chunks);
+  }
+
+  static contentLength(response: http.IncomingMessage): number {
+    const contentLength = response.headers['content-length'];
+    if (!contentLength) {
+      throw new Error('Empty response body when getting history');
+    }
+    return parseInt(contentLength);
+  }
+}

packages/worker/src/debug-replayer/inbound-interceptor.ts

@@ -0,0 +1,23 @@
+/**
+ * Debug replayer workflow inbound interceptors.
+ * Notify the runner when workflow starts or a signal is received, required for setting breakpoints on workflow tasks.
+ *
+ * @module
+ */
+import { WorkflowInterceptorsFactory } from '@temporalio/workflow';
+import { notifyRunner } from './workflow-notifier';
+
+export const interceptors: WorkflowInterceptorsFactory = () => ({
+  inbound: [
+    {
+      execute(input, next) {
+        notifyRunner();
+        return next(input);
+      },
+      handleSignal(input, next) {
+        notifyRunner();
+        return next(input);
+      },
+    },
+  ],
+});

packages/worker/src/debug-replayer/index.ts

@@ -0,0 +1,79 @@
+import worker_threads from 'node:worker_threads';
+import { temporal } from '@temporalio/proto';
+import { ReplayWorkerOptions } from '../worker-options';
+import { Worker } from '../worker';
+import { Client } from './client';
+
+let thread: worker_threads.Worker | undefined = undefined;
+
+async function run(options: ReplayWorkerOptions): Promise<void> {
+  const baseUrl = process.env.TEMPORAL_DEBUGGER_PLUGIN_URL;
+  if (!baseUrl) {
+    throw new Error('Missing TEMPORAL_DEBUGGER_PLUGIN_URL environment variable');
+  }
+  const client = new Client({ baseUrl });
+  const response = await client.get('history');
+  const history = temporal.api.history.v1.History.decode(
+    await Client.readAll(response),
+    Client.contentLength(response)
+  );
+
+  // Only create one per process.
+  // Not caring about globals here for to get simpler DX, this isn't meant for production use cases.
+  thread = thread || new worker_threads.Worker(require.resolve('./worker-thread'));
+
+  const rejectedPromise = new Promise<void>((_, reject) => {
+    // Set the global notifyRunner runner function that can be used from the workflow interceptors.
+    // The function makes an HTTP request to the runner in a separate thread and blocks the current thread until a
+    // response is received.
+    (globalThis as any).notifyRunner = (wftStartEventId: number) => {
+      const sab = new SharedArrayBuffer(4);
+      const responseBuffer = new Int32Array(sab);
+      thread?.postMessage({ eventId: wftStartEventId, responseBuffer });
+      Atomics.wait(responseBuffer, 0, 0);
+      if (responseBuffer[0] === 2) {
+        // Error occurred (logged by worker thread)
+        reject(new Error('Failed to call runner back'));
+      }
+    };
+  });
+
+  await Promise.race([
+    rejectedPromise,
+    Worker.runReplayHistory(
+      {
+        ...options,
+        interceptors: {
+          ...options.interceptors,
+          workflowModules: [
+            // Inbound goes first so user can set breakpoints in own inbound interceptors.
+            require.resolve('./inbound-interceptor'),
+            ...(options.interceptors?.workflowModules ?? []),
+            // Outbound goes last - notifies the runner in the finally block, user-provided outbound interceptors are
+            // resumed after the interceptor methods resolve.
+            require.resolve('./outbound-interceptor'),
+          ],
+        },
+      },
+      history
+    ),
+  ]);
+}
+
+/**
+ * Start a replayer for debugging purposes.
+ *
+ * Use this method to integrate the replayer with external debuggers like the Temporal VS Code debbuger extension.
+ */
+export function startDebugReplayer(options: ReplayWorkerOptions): void {
+  run(options).then(
+    () => {
+      console.log('Replay completed successfully');
+      process.exit(0);
+    },
+    (err) => {
+      console.error('Replay failed', err);
+      process.exit(1);
+    }
+  );
+}

packages/worker/src/debug-replayer/outbound-interceptor.ts

@@ -0,0 +1,49 @@
+/**
+ * Debug replayer workflow outbound interceptors.
+ * Notify the runner when outbound operations resolve, required for setting breakpoints on workflow tasks.
+ *
+ * @module
+ */
+import { WorkflowInterceptorsFactory } from '@temporalio/workflow';
+import { notifyRunner } from './workflow-notifier';
+
+export const interceptors: WorkflowInterceptorsFactory = () => ({
+  outbound: [
+    {
+      async scheduleActivity(input, next) {
+        try {
+          return await next(input);
+        } finally {
+          notifyRunner();
+        }
+      },
+      async scheduleLocalActivity(input, next) {
+        try {
+          return await next(input);
+        } finally {
+          notifyRunner();
+        }
+      },
+      async startTimer(input, next) {
+        try {
+          return await next(input);
+        } finally {
+          notifyRunner();
+        }
+      },
+      async signalWorkflow(input, next) {
+        try {
+          return await next(input);
+        } finally {
+          notifyRunner();
+        }
+      },
+      async startChildWorkflowExecution(input, next) {
+        const [startPromise, completePromise] = await next(input);
+        startPromise.finally(notifyRunner);
+        completePromise.finally(notifyRunner);
+        return [startPromise, completePromise];
+      },
+    },
+  ],
+});

packages/worker/src/debug-replayer/worker-thread.ts

@@ -0,0 +1,54 @@
+/**
+ * Worker thread entrypoint used by ./index.ts to synchronously make an HTTP call to the "runner".
+ */
+import { isMainThread, parentPort as parentPortOrNull } from 'node:worker_threads';
+import { Client } from './client';
+
+/**
+ * Request from parent thread, the worker thread should signal a "runner" when it gets this request.
+ */
+export interface Request {
+  type: 'wft-started';
+  /**
+   * Event ID of the started request.
+   */
+  eventId: number;
+  /**
+   * Used to signal back that the request is complete.
+   */
+  responseBuffer: Int32Array;
+}
+
+if (isMainThread) {
+  throw new Error(`Imported ${__filename} from main thread`);
+}
+
+if (parentPortOrNull === null) {
+  throw new TypeError(`${__filename} got a null parentPort`);
+}
+
+// Create a new parentPort variable that is not nullable to please TS
+const parentPort = parentPortOrNull;
+
+const baseUrl = process.env.TEMPORAL_DEBUGGER_PLUGIN_URL;
+if (!baseUrl) {
+  throw new Error('Missing TEMPORAL_DEBUGGER_PLUGIN_URL environment variable');
+}
+const client = new Client({ baseUrl });
+
+parentPort.on('message', async (request: Request) => {
+  const { eventId, responseBuffer } = request;
+  try {
+    await client.post(
+      'current-wft-started',
+      { headers: { 'Content-Type': 'application/json' }, timeout: 5000 },
+      Buffer.from(JSON.stringify({ eventId }))
+    );
+    Atomics.store(responseBuffer, 0, 1);
+  } catch (err) {
+    console.error(err);
+    Atomics.store(responseBuffer, 0, 2);
+  } finally {
+    Atomics.notify(responseBuffer, 0, 1);
+  }
+});

packages/worker/src/debug-replayer/workflow-notifier.ts

@@ -0,0 +1,46 @@
+import { workflowInfo } from '@temporalio/workflow';
+
+class WorkflowNotifier {
+  static _instance?: WorkflowNotifier;
+
+  /**
+   * Access the singleton instance - one per workflow context
+   */
+  static instance(): WorkflowNotifier {
+    if (this._instance === undefined) {
+      this._instance = new this();
+    }
+    return this._instance;
+  }
+
+  private constructor() {
+    // Dear eslint,
+    // I left this empty to mark the constructor private, OK?
+    //
+    // Best regards,
+    // - An anonymous developer
+  }
+
+  lastNotifiedStartEvent = -1;
+
+  notifyRunner(): void {
+    const eventId = workflowInfo().historyLength;
+    if (this.lastNotifiedStartEvent >= eventId) return;
+    this.lastNotifiedStartEvent = eventId;
+    try {
+      // Use global `notifyRunner` function, should be injected outside of workflow context.
+      // Using globalThis.constructor.constructor, we break out of the workflow context to Node.js land.
+      const notifyRunner = globalThis.constructor.constructor('return notifyRunner')();
+      notifyRunner(eventId);
+    } catch {
+      // ignore
+    }
+  }
+}
+
+/**
+ * Notify a runner process when a workflow task is picked up
+ */
+export function notifyRunner(): void {
+  WorkflowNotifier.instance().notifyRunner();
+}

packages/worker/src/index.ts

@@ -40,3 +40,4 @@ export {
 } from './worker-options';
 export { WorkflowInboundLogInterceptor, workflowLogAttributes } from './workflow-log-interceptor';
 export { BundleOptions, bundleWorkflowCode, WorkflowBundleWithSourceMap } from './workflow/bundler';
+export { startDebugReplayer } from './debug-replayer';

packages/worker/src/worker.ts

@@ -432,6 +432,8 @@ export class Worker {
     workflowBundle: WorkflowBundleWithSourceMapAndFilename,
     compiledOptions: CompiledWorkerOptions
   ): Promise<WorkflowCreator> {
+    // This isn't required for vscode, only for Chrome Dev Tools which doesn't support debugging worker threads.
+    // We also rely on this in debug-replayer where we inject a global variable to be read from workflow context.
     if (compiledOptions.debugMode) {
       return await VMWorkflowCreator.create(workflowBundle, compiledOptions.isolateExecutionTimeoutMs);
     } else {