[rush] Logging enhancements

common/changes/@microsoft/rush/logging-enhancement_2024-09-19-00-28.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Make individual Rush log files available via the rush-serve-plugin server at the relative URL specified by \"logServePath\" option. Annotate operations sent over the WebSocket with the URLs of their log files.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/reviews/api/rush-lib.api.md

@@ -523,6 +523,15 @@ export interface ILaunchOptions {
     terminalProvider?: ITerminalProvider;
 }
 
+// @alpha
+export interface ILogFilePaths {
+    error: string;
+    jsonl: string;
+    jsonlFolder: string;
+    text: string;
+    textFolder: string;
+}
+
 // @beta (undocumented)
 export interface ILogger {
     emitError(error: Error): void;
@@ -553,6 +562,7 @@ export interface _INpmOptionsJson extends IPackageManagerOptionsJsonBase {
 export interface IOperationExecutionResult {
     readonly cobuildRunnerId: string | undefined;
     readonly error: Error | undefined;
+    readonly logFilePaths: ILogFilePaths | undefined;
     readonly nonCachedDurationMs: number | undefined;
     readonly operation: Operation;
     readonly status: OperationStatus;

libraries/rush-lib/src/index.ts

@@ -134,6 +134,7 @@ export type {
 } from './logic/operations/IOperationExecutionResult';
 export { type IOperationOptions, Operation } from './logic/operations/Operation';
 export { OperationStatus } from './logic/operations/OperationStatus';
+export type { ILogFilePaths } from './logic/operations/ProjectLogWritable';
 
 export {
   RushSession,

libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts

@@ -14,7 +14,11 @@ import { ProjectBuildCache } from '../buildCache/ProjectBuildCache';
 import { RushConstants } from '../RushConstants';
 import { type IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration';
 import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync';
-import { ProjectLogWritable } from './ProjectLogWritable';
+import {
+  initializeProjectLogFilesAsync,
+  getProjectLogFilePaths,
+  type ILogFilePaths
+} from './ProjectLogWritable';
 import type { CobuildConfiguration } from '../../api/CobuildConfiguration';
 import { DisjointSet } from '../cobuild/DisjointSet';
 import { PeriodicCallback } from './PeriodicCallback';
@@ -59,7 +63,7 @@ export interface IOperationBuildCacheContext {
 
   // Controls the log for the cache subsystem
   buildCacheTerminal: ITerminal | undefined;
-  buildCacheProjectLogWritable: ProjectLogWritable | undefined;
+  buildCacheTerminalWritable: TerminalWritable | undefined;
 
   periodicCallback: PeriodicCallback;
   cacheRestored: boolean;
@@ -145,7 +149,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
               cobuildLock: undefined,
               cobuildClusterId: undefined,
               buildCacheTerminal: undefined,
-              buildCacheProjectLogWritable: undefined,
+              buildCacheTerminalWritable: undefined,
               periodicCallback: new PeriodicCallback({
                 interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000
               }),
@@ -233,9 +237,9 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
         const runBeforeExecute = async (): Promise<OperationStatus | undefined> => {
           if (
             !buildCacheContext.buildCacheTerminal ||
-            buildCacheContext.buildCacheProjectLogWritable?.isOpen === false
+            buildCacheContext.buildCacheTerminalWritable?.isOpen === false
           ) {
-            // The ProjectLogWritable is does not exist or is closed, re-create one
+            // The writable does not exist or has been closed, re-create one
             // eslint-disable-next-line require-atomic-updates
             buildCacheContext.buildCacheTerminal = await this._createBuildCacheTerminalAsync({
               record,
@@ -317,7 +321,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
           //     has changed happens inside the hashing logic.
           //
 
-          const { errorLogPath } = ProjectLogWritable.getLogFilePaths({
+          const { error: errorLogPath } = getProjectLogFilePaths({
             project,
             logFilenameIdentifier: operationMetadataManager.logFilenameIdentifier
           });
@@ -438,7 +442,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
             // Save the metadata to disk
             const { logFilenameIdentifier } = operationMetadataManager;
             const { duration: durationInSeconds } = stopwatch;
-            const { logPath, errorLogPath, logChunksPath } = ProjectLogWritable.getLogFilePaths({
+            const {
+              text: logPath,
+              error: errorLogPath,
+              jsonl: logChunksPath
+            } = getProjectLogFilePaths({
               project,
               logFilenameIdentifier
             });
@@ -505,7 +513,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
             }
           }
         } finally {
-          buildCacheContext.buildCacheProjectLogWritable?.close();
+          buildCacheContext.buildCacheTerminalWritable?.close();
           buildCacheContext.periodicCallback.stop();
         }
       }
@@ -746,12 +754,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
     let cacheConsoleWritable: TerminalWritable;
     // This creates the writer, only do this if necessary.
     const collatedWriter: CollatedWriter = record.collatedWriter;
-    const cacheProjectLogWritable: ProjectLogWritable | undefined =
-      await this._tryGetBuildCacheProjectLogWritableAsync({
+    const cacheProjectLogWritable: TerminalWritable | undefined =
+      await this._tryGetBuildCacheTerminalWritableAsync({
         buildCacheContext,
         buildCacheEnabled,
         rushProject,
-        collatedTerminal: collatedWriter.terminal,
         logFilenameIdentifier
       });
 
@@ -788,30 +795,32 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
     return new Terminal(buildCacheTerminalProvider);
   }
 
-  private async _tryGetBuildCacheProjectLogWritableAsync({
+  private async _tryGetBuildCacheTerminalWritableAsync({
     buildCacheEnabled,
     rushProject,
     buildCacheContext,
-    collatedTerminal,
     logFilenameIdentifier
   }: {
     buildCacheEnabled: boolean | undefined;
     rushProject: RushConfigurationProject;
     buildCacheContext: IOperationBuildCacheContext;
-    collatedTerminal: CollatedTerminal;
     logFilenameIdentifier: string;
-  }): Promise<ProjectLogWritable | undefined> {
+  }): Promise<TerminalWritable | undefined> {
     // Only open the *.cache.log file(s) if the cache is enabled.
     if (!buildCacheEnabled) {
       return;
     }
 
-    buildCacheContext.buildCacheProjectLogWritable = await ProjectLogWritable.initializeAsync({
+    const logFilePaths: ILogFilePaths = getProjectLogFilePaths({
       project: rushProject,
-      terminal: collatedTerminal,
       logFilenameIdentifier: `${logFilenameIdentifier}.cache`
     });
-    return buildCacheContext.buildCacheProjectLogWritable;
+
+    buildCacheContext.buildCacheTerminalWritable = await initializeProjectLogFilesAsync({
+      logFilePaths
+    });
+
+    return buildCacheContext.buildCacheTerminalWritable;
   }
 }
 async function updateAdditionalContextAsync({

libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts

@@ -5,6 +5,7 @@ import type { StdioSummarizer } from '@rushstack/terminal';
 import type { OperationStatus } from './OperationStatus';
 import type { Operation } from './Operation';
 import type { IStopwatchResult } from '../../utilities/Stopwatch';
+import type { ILogFilePaths } from './ProjectLogWritable';
 
 /**
  * The `IOperationExecutionResult` interface represents the results of executing an {@link Operation}.
@@ -43,6 +44,10 @@ export interface IOperationExecutionResult {
    * The id of the runner which actually runs the building process in cobuild mode.
    */
   readonly cobuildRunnerId: string | undefined;
+  /**
+   * The paths to the log files, if applicable.
+   */
+  readonly logFilePaths: ILogFilePaths | undefined;
 }
 
 /**

libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts

@@ -23,7 +23,11 @@ import { OperationMetadataManager } from './OperationMetadataManager';
 import type { IPhase } from '../../api/CommandLineConfiguration';
 import type { RushConfigurationProject } from '../../api/RushConfigurationProject';
 import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider';
-import { ProjectLogWritable } from './ProjectLogWritable';
+import {
+  getProjectLogFilePaths,
+  type ILogFilePaths,
+  initializeProjectLogFilesAsync
+} from './ProjectLogWritable';
 
 export interface IOperationExecutionRecordContext {
   streamCollator: StreamCollator;
@@ -103,6 +107,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext {
   public readonly associatedProject: RushConfigurationProject | undefined;
   public readonly _operationMetadataManager: OperationMetadataManager | undefined;
 
+  public logFilePaths: ILogFilePaths | undefined;
+
   private readonly _context: IOperationExecutionRecordContext;
 
   private _collatedWriter: CollatedWriter | undefined = undefined;
@@ -121,13 +127,17 @@ export class OperationExecutionRecord implements IOperationRunnerContext {
     this.runner = runner;
     this.associatedPhase = associatedPhase;
     this.associatedProject = associatedProject;
-    if (operation.associatedPhase && operation.associatedProject) {
-      this._operationMetadataManager = new OperationMetadataManager({
-        phase: operation.associatedPhase,
-        rushProject: operation.associatedProject,
-        operation
-      });
-    }
+    this.logFilePaths = undefined;
+
+    this._operationMetadataManager =
+      associatedPhase && associatedProject
+        ? new OperationMetadataManager({
+            phase: associatedPhase,
+            rushProject: associatedProject,
+            operation
+          })
+        : undefined;
+
     this._context = context;
     this._status = operation.dependencies.size > 0 ? OperationStatus.Waiting : OperationStatus.Ready;
   }
@@ -199,35 +209,35 @@ export class OperationExecutionRecord implements IOperationRunnerContext {
   ): Promise<T> {
     const { associatedPhase, associatedProject, stdioSummarizer } = this;
     const { createLogFile, logFileSuffix = '' } = options;
-    const projectLogWritable: ProjectLogWritable | undefined =
+
+    const logFilePaths: ILogFilePaths | undefined =
       createLogFile && associatedProject && associatedPhase && this._operationMetadataManager
-        ? await ProjectLogWritable.initializeAsync({
+        ? getProjectLogFilePaths({
             project: associatedProject,
-            terminal: this.collatedWriter.terminal,
-            logFilenameIdentifier: `${this._operationMetadataManager.logFilenameIdentifier}${logFileSuffix}`,
-            enableChunkedOutput: true
+            logFilenameIdentifier: `${this._operationMetadataManager.logFilenameIdentifier}${logFileSuffix}`
           })
         : undefined;
+    this.logFilePaths = logFilePaths;
+
+    const projectLogWritable: TerminalWritable | undefined = logFilePaths
+      ? await initializeProjectLogFilesAsync({
+          logFilePaths,
+          enableChunkedOutput: true
+        })
+      : undefined;
 
     try {
       //#region OPERATION LOGGING
       // TERMINAL PIPELINE:
       //
       //                             +--> quietModeTransform? --> collatedWriter
       //                             |
-      // normalizeNewlineTransform --1--> stderrLineTransform --2--> removeColorsTransform --> projectLogWritable
+      // normalizeNewlineTransform --1--> stderrLineTransform --2--> projectLogWritable
       //                                                        |
       //                                                        +--> stdioSummarizer
       const destination: TerminalWritable = projectLogWritable
         ? new SplitterTransform({
-            destinations: [
-              new TextRewriterTransform({
-                destination: projectLogWritable,
-                removeColors: true,
-                normalizeNewlines: NewlineKind.OsDefault
-              }),
-              stdioSummarizer
-            ]
+            destinations: [projectLogWritable, stdioSummarizer]
           })
         : stdioSummarizer;