diff --git a/common/changes/@microsoft/rush/logging-enhancement_2024-09-19-00-28.json b/common/changes/@microsoft/rush/logging-enhancement_2024-09-19-00-28.json new file mode 100644 index 00000000000..5341e2283a2 --- /dev/null +++ b/common/changes/@microsoft/rush/logging-enhancement_2024-09-19-00-28.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Make individual Rush log files available via the rush-serve-plugin server at the relative URL specified by \"logServePath\" option. Annotate operations sent over the WebSocket with the URLs of their log files.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 9e659294629..846d85eec1a 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -523,6 +523,15 @@ export interface ILaunchOptions { terminalProvider?: ITerminalProvider; } +// @alpha +export interface ILogFilePaths { + error: string; + jsonl: string; + jsonlFolder: string; + text: string; + textFolder: string; +} + // @beta (undocumented) export interface ILogger { emitError(error: Error): void; @@ -553,6 +562,7 @@ export interface _INpmOptionsJson extends IPackageManagerOptionsJsonBase { export interface IOperationExecutionResult { readonly cobuildRunnerId: string | undefined; readonly error: Error | undefined; + readonly logFilePaths: ILogFilePaths | undefined; readonly nonCachedDurationMs: number | undefined; readonly operation: Operation; readonly status: OperationStatus; diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index 63e7e812116..f7d2e9a2278 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -134,6 +134,7 @@ export type { } from './logic/operations/IOperationExecutionResult'; export { type IOperationOptions, Operation } from './logic/operations/Operation'; export { OperationStatus } from './logic/operations/OperationStatus'; +export type { ILogFilePaths } from './logic/operations/ProjectLogWritable'; export { RushSession, diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index a29db3d9c97..a87f194ab5e 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -14,7 +14,11 @@ import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; import { RushConstants } from '../RushConstants'; import { type IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; -import { ProjectLogWritable } from './ProjectLogWritable'; +import { + initializeProjectLogFilesAsync, + getProjectLogFilePaths, + type ILogFilePaths +} from './ProjectLogWritable'; import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; import { DisjointSet } from '../cobuild/DisjointSet'; import { PeriodicCallback } from './PeriodicCallback'; @@ -59,7 +63,7 @@ export interface IOperationBuildCacheContext { // Controls the log for the cache subsystem buildCacheTerminal: ITerminal | undefined; - buildCacheProjectLogWritable: ProjectLogWritable | undefined; + buildCacheTerminalWritable: TerminalWritable | undefined; periodicCallback: PeriodicCallback; cacheRestored: boolean; @@ -145,7 +149,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { cobuildLock: undefined, cobuildClusterId: undefined, buildCacheTerminal: undefined, - buildCacheProjectLogWritable: undefined, + buildCacheTerminalWritable: undefined, periodicCallback: new PeriodicCallback({ interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000 }), @@ -233,9 +237,9 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const runBeforeExecute = async (): Promise => { if ( !buildCacheContext.buildCacheTerminal || - buildCacheContext.buildCacheProjectLogWritable?.isOpen === false + buildCacheContext.buildCacheTerminalWritable?.isOpen === false ) { - // The ProjectLogWritable is does not exist or is closed, re-create one + // The writable does not exist or has been closed, re-create one // eslint-disable-next-line require-atomic-updates buildCacheContext.buildCacheTerminal = await this._createBuildCacheTerminalAsync({ record, @@ -317,7 +321,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // has changed happens inside the hashing logic. // - const { errorLogPath } = ProjectLogWritable.getLogFilePaths({ + const { error: errorLogPath } = getProjectLogFilePaths({ project, logFilenameIdentifier: operationMetadataManager.logFilenameIdentifier }); @@ -438,7 +442,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // Save the metadata to disk const { logFilenameIdentifier } = operationMetadataManager; const { duration: durationInSeconds } = stopwatch; - const { logPath, errorLogPath, logChunksPath } = ProjectLogWritable.getLogFilePaths({ + const { + text: logPath, + error: errorLogPath, + jsonl: logChunksPath + } = getProjectLogFilePaths({ project, logFilenameIdentifier }); @@ -505,7 +513,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } } finally { - buildCacheContext.buildCacheProjectLogWritable?.close(); + buildCacheContext.buildCacheTerminalWritable?.close(); buildCacheContext.periodicCallback.stop(); } } @@ -746,12 +754,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { let cacheConsoleWritable: TerminalWritable; // This creates the writer, only do this if necessary. const collatedWriter: CollatedWriter = record.collatedWriter; - const cacheProjectLogWritable: ProjectLogWritable | undefined = - await this._tryGetBuildCacheProjectLogWritableAsync({ + const cacheProjectLogWritable: TerminalWritable | undefined = + await this._tryGetBuildCacheTerminalWritableAsync({ buildCacheContext, buildCacheEnabled, rushProject, - collatedTerminal: collatedWriter.terminal, logFilenameIdentifier }); @@ -788,30 +795,32 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return new Terminal(buildCacheTerminalProvider); } - private async _tryGetBuildCacheProjectLogWritableAsync({ + private async _tryGetBuildCacheTerminalWritableAsync({ buildCacheEnabled, rushProject, buildCacheContext, - collatedTerminal, logFilenameIdentifier }: { buildCacheEnabled: boolean | undefined; rushProject: RushConfigurationProject; buildCacheContext: IOperationBuildCacheContext; - collatedTerminal: CollatedTerminal; logFilenameIdentifier: string; - }): Promise { + }): Promise { // Only open the *.cache.log file(s) if the cache is enabled. if (!buildCacheEnabled) { return; } - buildCacheContext.buildCacheProjectLogWritable = await ProjectLogWritable.initializeAsync({ + const logFilePaths: ILogFilePaths = getProjectLogFilePaths({ project: rushProject, - terminal: collatedTerminal, logFilenameIdentifier: `${logFilenameIdentifier}.cache` }); - return buildCacheContext.buildCacheProjectLogWritable; + + buildCacheContext.buildCacheTerminalWritable = await initializeProjectLogFilesAsync({ + logFilePaths + }); + + return buildCacheContext.buildCacheTerminalWritable; } } async function updateAdditionalContextAsync({ diff --git a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts index ea7f4476daa..578c3c0fe8b 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts @@ -5,6 +5,7 @@ import type { StdioSummarizer } from '@rushstack/terminal'; import type { OperationStatus } from './OperationStatus'; import type { Operation } from './Operation'; import type { IStopwatchResult } from '../../utilities/Stopwatch'; +import type { ILogFilePaths } from './ProjectLogWritable'; /** * The `IOperationExecutionResult` interface represents the results of executing an {@link Operation}. @@ -43,6 +44,10 @@ export interface IOperationExecutionResult { * The id of the runner which actually runs the building process in cobuild mode. */ readonly cobuildRunnerId: string | undefined; + /** + * The paths to the log files, if applicable. + */ + readonly logFilePaths: ILogFilePaths | undefined; } /** diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 191907766ae..efc93a3e39d 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -23,7 +23,11 @@ import { OperationMetadataManager } from './OperationMetadataManager'; import type { IPhase } from '../../api/CommandLineConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; -import { ProjectLogWritable } from './ProjectLogWritable'; +import { + getProjectLogFilePaths, + type ILogFilePaths, + initializeProjectLogFilesAsync +} from './ProjectLogWritable'; export interface IOperationExecutionRecordContext { streamCollator: StreamCollator; @@ -103,6 +107,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext { public readonly associatedProject: RushConfigurationProject | undefined; public readonly _operationMetadataManager: OperationMetadataManager | undefined; + public logFilePaths: ILogFilePaths | undefined; + private readonly _context: IOperationExecutionRecordContext; private _collatedWriter: CollatedWriter | undefined = undefined; @@ -121,13 +127,17 @@ export class OperationExecutionRecord implements IOperationRunnerContext { this.runner = runner; this.associatedPhase = associatedPhase; this.associatedProject = associatedProject; - if (operation.associatedPhase && operation.associatedProject) { - this._operationMetadataManager = new OperationMetadataManager({ - phase: operation.associatedPhase, - rushProject: operation.associatedProject, - operation - }); - } + this.logFilePaths = undefined; + + this._operationMetadataManager = + associatedPhase && associatedProject + ? new OperationMetadataManager({ + phase: associatedPhase, + rushProject: associatedProject, + operation + }) + : undefined; + this._context = context; this._status = operation.dependencies.size > 0 ? OperationStatus.Waiting : OperationStatus.Ready; } @@ -199,15 +209,22 @@ export class OperationExecutionRecord implements IOperationRunnerContext { ): Promise { const { associatedPhase, associatedProject, stdioSummarizer } = this; const { createLogFile, logFileSuffix = '' } = options; - const projectLogWritable: ProjectLogWritable | undefined = + + const logFilePaths: ILogFilePaths | undefined = createLogFile && associatedProject && associatedPhase && this._operationMetadataManager - ? await ProjectLogWritable.initializeAsync({ + ? getProjectLogFilePaths({ project: associatedProject, - terminal: this.collatedWriter.terminal, - logFilenameIdentifier: `${this._operationMetadataManager.logFilenameIdentifier}${logFileSuffix}`, - enableChunkedOutput: true + logFilenameIdentifier: `${this._operationMetadataManager.logFilenameIdentifier}${logFileSuffix}` }) : undefined; + this.logFilePaths = logFilePaths; + + const projectLogWritable: TerminalWritable | undefined = logFilePaths + ? await initializeProjectLogFilesAsync({ + logFilePaths, + enableChunkedOutput: true + }) + : undefined; try { //#region OPERATION LOGGING @@ -215,19 +232,12 @@ export class OperationExecutionRecord implements IOperationRunnerContext { // // +--> quietModeTransform? --> collatedWriter // | - // normalizeNewlineTransform --1--> stderrLineTransform --2--> removeColorsTransform --> projectLogWritable + // normalizeNewlineTransform --1--> stderrLineTransform --2--> projectLogWritable // | // +--> stdioSummarizer const destination: TerminalWritable = projectLogWritable ? new SplitterTransform({ - destinations: [ - new TextRewriterTransform({ - destination: projectLogWritable, - removeColors: true, - normalizeNewlines: NewlineKind.OsDefault - }), - stdioSummarizer - ] + destinations: [projectLogWritable, stdioSummarizer] }) : stdioSummarizer; diff --git a/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts b/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts index c15a1f6ee5a..cac65421518 100644 --- a/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts +++ b/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts @@ -1,165 +1,154 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { FileSystem, FileWriter, InternalError } from '@rushstack/node-core-library'; -import { TerminalChunkKind, TerminalWritable, type ITerminalChunk } from '@rushstack/terminal'; -import type { CollatedTerminal } from '@rushstack/stream-collator'; +import { FileSystem, FileWriter, InternalError, NewlineKind } from '@rushstack/node-core-library'; +import { + SplitterTransform, + TerminalChunkKind, + TerminalWritable, + TextRewriterTransform, + type ITerminalChunk +} from '@rushstack/terminal'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { PackageNameParsers } from '../../api/PackageNameParsers'; import { RushConstants } from '../RushConstants'; export interface IProjectLogWritableOptions { - project: RushConfigurationProject; - terminal: CollatedTerminal; - logFilenameIdentifier: string; + logFilePaths: ILogFilePaths; enableChunkedOutput?: boolean; } +export interface ILogFileNames { + textFileName: string; + jsonlFileName: string; + errorFileName: string; +} + +/** + * Information about the log files for an operation. + * + * @alpha + */ export interface ILogFilePaths { - logFolderPath: string; - logChunksFolderPath: string; - - logPath: string; - logChunksPath: string; - errorLogPath: string; - relativeLogPath: string; - relativeErrorLogPath: string; - relativeLogChunksPath: string; + /** + * The absolute path to the folder containing the text log files. + * Provided as a convenience since it is an intermediary value of producing the text log file path. + */ + textFolder: string; + /** + * The absolute path to the folder containing the JSONL log files. + * Provided as a convenience since it is an intermediary value of producing the jsonl log file path. + */ + jsonlFolder: string; + + /** + * The absolute path to the merged (interleaved stdout and stderr) text log. + * ANSI escape codes have been stripped. + */ + text: string; + /** + * The absolute path to the stderr text log. + * ANSI escape codes have been stripped. + */ + error: string; + /** + * The absolute path to the JSONL log. ANSI escape codes are left intact to be able to reproduce the console output. + */ + jsonl: string; } export interface IGetLogFilePathsOptions { - project: RushConfigurationProject; + project: Pick; logFilenameIdentifier: string; } const LOG_CHUNKS_FOLDER_RELATIVE_PATH: string = `${RushConstants.projectRushFolderName}/${RushConstants.rushTempFolderName}/chunked-rush-logs`; -export class ProjectLogWritable extends TerminalWritable { - private readonly _terminal: CollatedTerminal; - +/** + * A terminal stream that writes all log chunks to a JSONL format so they can be faithfully reconstructed + * during build cache restores. This is used for adding warning + error messages in cobuilds where the original + * logs cannot be completely restored from the existing `all.log` and `error.log` files. + * + * Example output: + * libraries/rush-lib/.rush/temp/operations/rush-lib._phase_build.chunks.jsonl + * ``` + * {"kind":"O","text":"Invoking: heft run --only build -- --clean \n"} + * {"kind":"O","text":" ---- build started ---- \n"} + * {"kind":"O","text":"[build:clean] Deleted 0 files and 5 folders\n"} + * {"kind":"O","text":"[build:typescript] Using TypeScript version 5.4.2\n"} + * {"kind":"O","text":"[build:lint] Using ESLint version 8.57.0\n"} + * {"kind":"E","text":"[build:lint] Warning: libraries/rush-lib/src/logic/operations/LogChunksWritable.ts:15:7 - (@typescript-eslint/typedef) Expected test to have a type annotation.\n"} + * {"kind":"E","text":"[build:lint] Warning: libraries/rush-lib/src/logic/operations/LogChunksWritable.ts:15:7 - (@typescript-eslint/no-unused-vars) 'test' is assigned a value but never used.\n"} + * {"kind":"O","text":"[build:typescript] Copied 1138 folders or files and linked 0 files\n"} + * {"kind":"O","text":"[build:webpack] Using Webpack version 5.82.1\n"} + * {"kind":"O","text":"[build:webpack] Running Webpack compilation\n"} + * {"kind":"O","text":"[build:api-extractor] Using API Extractor version 7.43.1\n"} + * {"kind":"O","text":"[build:api-extractor] Analysis will use the bundled TypeScript version 5.4.2\n"} + * {"kind":"O","text":"[build:copy-mock-flush-telemetry-plugin] Copied 1260 folders or files and linked 5 files\n"} + * {"kind":"O","text":" ---- build finished (6.856s) ---- \n"} + * {"kind":"O","text":"-------------------- Finished (6.858s) --------------------\n"} + * ``` + */ +export class JsonLFileWritable extends TerminalWritable { public readonly logPath: string; - public readonly errorLogPath: string; - public readonly logChunksPath: string; - public readonly relativeLogPath: string; - public readonly relativeErrorLogPath: string; - public readonly relativeLogChunksPath: string; - private _logWriter: FileWriter | undefined = undefined; - private _errorLogWriter: FileWriter | undefined = undefined; + private _writer: FileWriter | undefined; - /** - * A new terminal stream that writes all log chunks to a JSON format so they can be faithfully reconstructed - * during build cache restores. This is used for adding warning + error messages in cobuilds where the original - * logs cannot be completely restored from the existing `all.log` and `error.log` files. - * - * Example output: - * libraries/rush-lib/.rush/temp/operations/rush-lib._phase_build.chunks.jsonl - * ``` - * {"kind":"O","text":"Invoking: heft run --only build -- --clean \n"} - * {"kind":"O","text":" ---- build started ---- \n"} - * {"kind":"O","text":"[build:clean] Deleted 0 files and 5 folders\n"} - * {"kind":"O","text":"[build:typescript] Using TypeScript version 5.4.2\n"} - * {"kind":"O","text":"[build:lint] Using ESLint version 8.57.0\n"} - * {"kind":"E","text":"[build:lint] Warning: libraries/rush-lib/src/logic/operations/LogChunksWritable.ts:15:7 - (@typescript-eslint/typedef) Expected test to have a type annotation.\n"} - * {"kind":"E","text":"[build:lint] Warning: libraries/rush-lib/src/logic/operations/LogChunksWritable.ts:15:7 - (@typescript-eslint/no-unused-vars) 'test' is assigned a value but never used.\n"} - * {"kind":"O","text":"[build:typescript] Copied 1138 folders or files and linked 0 files\n"} - * {"kind":"O","text":"[build:webpack] Using Webpack version 5.82.1\n"} - * {"kind":"O","text":"[build:webpack] Running Webpack compilation\n"} - * {"kind":"O","text":"[build:api-extractor] Using API Extractor version 7.43.1\n"} - * {"kind":"O","text":"[build:api-extractor] Analysis will use the bundled TypeScript version 5.4.2\n"} - * {"kind":"O","text":"[build:copy-mock-flush-telemetry-plugin] Copied 1260 folders or files and linked 5 files\n"} - * {"kind":"O","text":" ---- build finished (6.856s) ---- \n"} - * {"kind":"O","text":"-------------------- Finished (6.858s) --------------------\n"} - * ``` - */ - private _chunkWriter: FileWriter | undefined; - - private _enableChunkedOutput: boolean; - - private constructor( - terminal: CollatedTerminal, - enableChunkedOutput: boolean, - { - logPath, - errorLogPath, - logChunksPath, - relativeLogPath, - relativeErrorLogPath, - relativeLogChunksPath - }: ILogFilePaths - ) { + public constructor(logPath: string) { super(); - this._terminal = terminal; - this._enableChunkedOutput = enableChunkedOutput; this.logPath = logPath; - this.errorLogPath = errorLogPath; - this.logChunksPath = logChunksPath; - this.relativeLogPath = relativeLogPath; - this.relativeErrorLogPath = relativeErrorLogPath; - this.relativeLogChunksPath = relativeLogChunksPath; - this._logWriter = FileWriter.open(logPath); - this._chunkWriter = FileWriter.open(logChunksPath); + this._writer = FileWriter.open(logPath); } - public static async initializeAsync({ - project, - terminal, - logFilenameIdentifier, - enableChunkedOutput = false - }: IProjectLogWritableOptions): Promise { - const logFilePaths: ILogFilePaths = ProjectLogWritable.getLogFilePaths({ - project, - logFilenameIdentifier - }); - - const { logFolderPath, logChunksFolderPath, logPath, errorLogPath, logChunksPath } = logFilePaths; - await Promise.all([ - FileSystem.ensureFolderAsync(logFolderPath), - FileSystem.ensureFolderAsync(logChunksFolderPath), - FileSystem.deleteFileAsync(logPath), - FileSystem.deleteFileAsync(errorLogPath), - FileSystem.deleteFileAsync(logChunksPath) - ]); - - return new ProjectLogWritable(terminal, enableChunkedOutput, logFilePaths); + // Override writeChunk function to throw custom error + public writeChunk(chunk: ITerminalChunk): void { + if (!this._writer) { + throw new InternalError(`Log writer was closed for ${this.logPath}`); + } + // Stderr can always get written to a error log writer + super.writeChunk(chunk); } - public static getLogFilePaths({ - project: { projectFolder, packageName }, - logFilenameIdentifier - }: IGetLogFilePathsOptions): ILogFilePaths { - const unscopedProjectName: string = PackageNameParsers.permissive.getUnscopedName(packageName); - const logFileBaseName: string = `${unscopedProjectName}.${logFilenameIdentifier}`; - - const logFolderPath: string = `${projectFolder}/${RushConstants.rushLogsFolderName}`; - const logChunksFolderPath: string = `${projectFolder}/${LOG_CHUNKS_FOLDER_RELATIVE_PATH}`; + protected onWriteChunk(chunk: ITerminalChunk): void { + if (!this._writer) { + throw new InternalError(`Log writer was closed for ${this.logPath}`); + } + this._writer.write(JSON.stringify(chunk) + '\n'); + } - const logFileBasePath: string = `${RushConstants.rushLogsFolderName}/${logFileBaseName}`; - const chunkLogFileBasePath: string = `${LOG_CHUNKS_FOLDER_RELATIVE_PATH}/${logFileBaseName}`; + protected onClose(): void { + if (this._writer) { + try { + this._writer.close(); + } catch (error) { + throw new InternalError('Failed to close file handle for ' + this._writer.filePath); + } + this._writer = undefined; + } + } +} - const relativeLogPath: string = `${logFileBasePath}.log`; - const relativeErrorLogPath: string = `${logFileBasePath}.error.log`; - const relativeLogChunksPath: string = `${chunkLogFileBasePath}.chunks.jsonl`; +/** + * A terminal stream that writes two text log files: one with interleaved stdout and stderr, and one with just stderr. + */ +export class SplitLogFileWritable extends TerminalWritable { + public readonly logPath: string; + public readonly errorLogPath: string; - const logPath: string = `${projectFolder}/${relativeLogPath}`; - const errorLogPath: string = `${projectFolder}/${relativeErrorLogPath}`; - const logChunksPath: string = `${projectFolder}/${relativeLogChunksPath}`; + private _logWriter: FileWriter | undefined = undefined; + private _errorLogWriter: FileWriter | undefined = undefined; - return { - logFolderPath, - logChunksFolderPath, + public constructor(logPath: string, errorLogPath: string) { + super(); - logPath, - errorLogPath, - logChunksPath, + this.logPath = logPath; + this.errorLogPath = errorLogPath; - relativeLogChunksPath, - relativeLogPath, - relativeErrorLogPath - }; + this._logWriter = FileWriter.open(logPath); + this._errorLogWriter = FileWriter.open(errorLogPath); } // Override writeChunk function to throw custom error @@ -178,13 +167,6 @@ export class ProjectLogWritable extends TerminalWritable { // Both stderr and stdout get written to *..log this._logWriter.write(chunk.text); - if (this._enableChunkedOutput) { - if (!this._chunkWriter) { - throw new InternalError('Chunked output file was closed'); - } - this._chunkWriter.write(JSON.stringify(chunk) + '\n'); - } - if (chunk.kind === TerminalChunkKind.Stderr) { // Only stderr gets written to *..error.log if (!this._errorLogWriter) { @@ -199,7 +181,7 @@ export class ProjectLogWritable extends TerminalWritable { try { this._logWriter.close(); } catch (error) { - this._terminal.writeStderrLine('Failed to close file handle for ' + this._logWriter.filePath); + throw new InternalError('Failed to close file handle for ' + this._logWriter.filePath); } this._logWriter = undefined; } @@ -208,18 +190,118 @@ export class ProjectLogWritable extends TerminalWritable { try { this._errorLogWriter.close(); } catch (error) { - this._terminal.writeStderrLine('Failed to close file handle for ' + this._errorLogWriter.filePath); + throw new InternalError('Failed to close file handle for ' + this._errorLogWriter.filePath); } this._errorLogWriter = undefined; } + } +} - if (this._chunkWriter) { - try { - this._chunkWriter.close(); - } catch (error) { - this._terminal.writeStderrLine('Failed to close file handle for ' + this._chunkWriter.filePath); - } - this._chunkWriter = undefined; - } +/** + * Initializes the project log files for a project. Produces a combined log file, an error log file, and optionally a + * chunks file that can be used to reconstrct the original console output. + * @param options - The options to initialize the project log files. + * @returns The terminal writable stream that will write to the log files. + */ +export async function initializeProjectLogFilesAsync( + options: IProjectLogWritableOptions +): Promise { + const { logFilePaths, enableChunkedOutput = false } = options; + + const { + textFolder: logFolderPath, + jsonlFolder: jsonlFolderPath, + text: logPath, + error: errorLogPath, + jsonl: jsonlPath + } = logFilePaths; + await Promise.all([ + FileSystem.ensureFolderAsync(logFolderPath), + enableChunkedOutput && FileSystem.ensureFolderAsync(jsonlFolderPath), + FileSystem.deleteFileAsync(logPath), + FileSystem.deleteFileAsync(errorLogPath), + FileSystem.deleteFileAsync(jsonlPath) + ]); + + const splitLog: TerminalWritable = new TextRewriterTransform({ + destination: new SplitLogFileWritable(logPath, errorLogPath), + removeColors: true, + normalizeNewlines: NewlineKind.OsDefault + }); + + if (enableChunkedOutput) { + const chunksFile: JsonLFileWritable = new JsonLFileWritable(jsonlPath); + const splitter: SplitterTransform = new SplitterTransform({ + destinations: [splitLog, chunksFile] + }); + return splitter; } + + return splitLog; +} + +/** + * @internal + * + * @param packageName - The raw package name + * @param logFilenameIdentifier - The identifier to append to the log file name (typically the phase name) + * @returns The base names of the log files + */ +export function getLogfileBaseNames(packageName: string, logFilenameIdentifier: string): ILogFileNames { + const unscopedProjectName: string = PackageNameParsers.permissive.getUnscopedName(packageName); + const logFileBaseName: string = `${unscopedProjectName}.${logFilenameIdentifier}`; + + return { + textFileName: `${logFileBaseName}.log`, + jsonlFileName: `${logFileBaseName}.chunks.jsonl`, + errorFileName: `${logFileBaseName}.error.log` + }; +} + +/** + * @internal + * + * @param projectFolder - The absolute path of the project folder + * @returns The absolute paths of the log folders for regular and chunked logs + */ +export function getProjectLogFolders( + projectFolder: string +): Pick { + const textFolder: string = `${projectFolder}/${RushConstants.rushLogsFolderName}`; + const jsonlFolder: string = `${projectFolder}/${LOG_CHUNKS_FOLDER_RELATIVE_PATH}`; + + return { textFolder, jsonlFolder }; +} + +/** + * @internal + * + * @param options - The options to get the log file paths + * @returns All information about log file paths for the project and log identifier + */ +export function getProjectLogFilePaths(options: IGetLogFilePathsOptions): ILogFilePaths { + const { + project: { projectFolder, packageName }, + logFilenameIdentifier + } = options; + + const { textFolder, jsonlFolder } = getProjectLogFolders(projectFolder); + const { + textFileName: textLog, + jsonlFileName: jsonlLog, + errorFileName: errorLog + } = getLogfileBaseNames(packageName, logFilenameIdentifier); + + const textPath: string = `${textFolder}/${textLog}`; + const errorPath: string = `${textFolder}/${errorLog}`; + const jsonlPath: string = `${jsonlFolder}/${jsonlLog}`; + + return { + textFolder, + jsonlFolder, + + text: textPath, + error: errorPath, + jsonl: jsonlPath + }; } diff --git a/rush-plugins/rush-serve-plugin/src/RushProjectServeConfigFile.ts b/rush-plugins/rush-serve-plugin/src/RushProjectServeConfigFile.ts index 62322db213e..0182025bc19 100644 --- a/rush-plugins/rush-serve-plugin/src/RushProjectServeConfigFile.ts +++ b/rush-plugins/rush-serve-plugin/src/RushProjectServeConfigFile.ts @@ -57,7 +57,7 @@ export class RushServeConfiguration { projects: Iterable, terminal: ITerminal, workspaceRoutingRules: Iterable - ): Promise> { + ): Promise { const rules: IRoutingRule[] = Array.from(workspaceRoutingRules); await Async.forEachAsync( diff --git a/rush-plugins/rush-serve-plugin/src/RushServePlugin.ts b/rush-plugins/rush-serve-plugin/src/RushServePlugin.ts index a3d2b7bfbd2..08be11d31fb 100644 --- a/rush-plugins/rush-serve-plugin/src/RushServePlugin.ts +++ b/rush-plugins/rush-serve-plugin/src/RushServePlugin.ts @@ -37,6 +37,11 @@ export interface IRushServePluginOptions { */ portParameterLongName?: string | undefined; + /** + * The URL path at which to host Rush log files. If not specified, log files will not be served. + */ + logServePath?: string | undefined; + /** * Routing rules for files that are associated with the entire workspace, rather than a single project (e.g. files output by Rush plugins). */ @@ -52,12 +57,14 @@ export class RushServePlugin implements IRushPlugin { private readonly _phasedCommands: Set; private readonly _portParameterLongName: string | undefined; private readonly _globalRoutingRules: IGlobalRoutingRuleJson[]; + private readonly _logServePath: string | undefined; private readonly _buildStatusWebSocketPath: string | undefined; public constructor(options: IRushServePluginOptions) { this._phasedCommands = new Set(options.phasedCommands); this._portParameterLongName = options.portParameterLongName; this._globalRoutingRules = options.globalRouting ?? []; + this._logServePath = options.logServePath; this._buildStatusWebSocketPath = options.buildStatusWebSocketPath; } @@ -84,6 +91,7 @@ export class RushServePlugin implements IRushPlugin { rushConfiguration, command, portParameterLongName: this._portParameterLongName, + logServePath: this._logServePath, globalRoutingRules, buildStatusWebSocketPath: this._buildStatusWebSocketPath }); diff --git a/rush-plugins/rush-serve-plugin/src/api.types.ts b/rush-plugins/rush-serve-plugin/src/api.types.ts index 93882996945..40497b4bc7b 100644 --- a/rush-plugins/rush-serve-plugin/src/api.types.ts +++ b/rush-plugins/rush-serve-plugin/src/api.types.ts @@ -8,6 +8,23 @@ import type { OperationStatus } from '@rushstack/rush-sdk'; */ export type ReadableOperationStatus = keyof typeof OperationStatus; +export interface ILogFileURLs { + /** + * The relative URL to the merged (interleaved stdout and stderr) text log. + */ + text: string; + + /** + * The relative URL to the stderr log file. + */ + error: string; + + /** + * The relative URL to the JSONL log file. + */ + jsonl: string; +} + /** * Information about an operation in the graph. */ @@ -41,6 +58,11 @@ export interface IOperationInfo { */ status: ReadableOperationStatus; + /** + * The URLs to the log files, if applicable. + */ + logFileURLs: ILogFileURLs | undefined; + /** * The start time of the operation, if it has started, in milliseconds. Not wall clock time. */ @@ -80,6 +102,7 @@ export interface IWebSocketBeforeExecuteEventMessage { */ export interface IWebSocketAfterExecuteEventMessage { event: 'after-execute'; + operations: IOperationInfo[]; status: ReadableOperationStatus; } diff --git a/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts b/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts index e906350465f..60682d6fb00 100644 --- a/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts +++ b/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts @@ -25,8 +25,10 @@ import { type ICreateOperationsContext, type IOperationExecutionResult, OperationStatus, - type IExecutionResult + type IExecutionResult, + type ILogFilePaths } from '@rushstack/rush-sdk'; +import { getProjectLogFolders } from '@rushstack/rush-sdk/lib/logic/operations/ProjectLogWritable'; import type { CommandLineStringParameter } from '@rushstack/ts-command-line'; import { PLUGIN_NAME } from './constants'; @@ -41,7 +43,8 @@ import type { IWebSocketSyncEventMessage, ReadableOperationStatus, IWebSocketCommandMessage, - IRushSessionInfo + IRushSessionInfo, + ILogFileURLs } from './api.types'; export interface IPhasedCommandHandlerOptions { @@ -49,6 +52,7 @@ export interface IPhasedCommandHandlerOptions { rushConfiguration: RushConfiguration; command: IPhasedCommand; portParameterLongName: string | undefined; + logServePath: string | undefined; globalRoutingRules: IRoutingRule[]; buildStatusWebSocketPath: string | undefined; } @@ -153,12 +157,33 @@ export async function phasedCommandHandler(options: IPhasedCommandHandlerOptions const serveConfig: RushServeConfiguration = new RushServeConfiguration(); - const routingRules: Iterable = await serveConfig.loadProjectConfigsAsync( + const routingRules: IRoutingRule[] = await serveConfig.loadProjectConfigsAsync( selectedProjects, logger.terminal, globalRoutingRules ); + const { logServePath } = options; + if (logServePath) { + for (const project of selectedProjects) { + const projectLogServePath: string = getLogServePathForProject(logServePath, project.packageName); + + routingRules.push({ + type: 'folder', + diskPath: getProjectLogFolders(project.projectFolder).textFolder, + servePath: projectLogServePath, + immutable: false + }); + + routingRules.push({ + type: 'folder', + diskPath: getProjectLogFolders(project.projectFolder).jsonlFolder, + servePath: projectLogServePath, + immutable: false + }); + } + } + const fileRoutingRules: Map = new Map(); const wbnRegex: RegExp = /\.wbn$/i; @@ -274,6 +299,27 @@ function tryEnableBuildStatusWebSocketServer( [OperationStatus.NoOp]: 'NoOp' }; + const { logServePath } = options; + + function convertToLogFileUrls( + logFilePaths: ILogFilePaths | undefined, + packageName: string + ): ILogFileURLs | undefined { + if (!logFilePaths || !logServePath) { + return; + } + + const projectLogServePath: string = getLogServePathForProject(logServePath, packageName); + + const logFileUrls: ILogFileURLs = { + text: `${projectLogServePath}${logFilePaths.text.slice(logFilePaths.textFolder.length)}`, + error: `${projectLogServePath}${logFilePaths.error.slice(logFilePaths.textFolder.length)}`, + jsonl: `${projectLogServePath}${logFilePaths.jsonl.slice(logFilePaths.jsonlFolder.length)}` + }; + + return logFileUrls; + } + /** * Maps the internal Rush record down to a subset that is JSON-friendly and human readable. */ @@ -285,9 +331,11 @@ function tryEnableBuildStatusWebSocketServer( return; } + const { packageName } = associatedProject; + return { name, - packageName: associatedProject.packageName, + packageName, phaseName: associatedPhase.name, silent: !!runner.silent, @@ -295,7 +343,9 @@ function tryEnableBuildStatusWebSocketServer( status: readableStatusFromStatus[record.status], startTime: record.stopwatch.startTime, - endTime: record.stopwatch.endTime + endTime: record.stopwatch.endTime, + + logFileURLs: convertToLogFileUrls(record.logFilePaths, packageName) }; } @@ -356,8 +406,10 @@ function tryEnableBuildStatusWebSocketServer( hooks.afterExecuteOperations.tap(PLUGIN_NAME, (result: IExecutionResult): void => { buildStatus = readableStatusFromStatus[result.status]; + const infos: IOperationInfo[] = convertToOperationInfoArray(result.operationResults.values() ?? []); const afterExecuteMessage: IWebSocketAfterExecuteEventMessage = { event: 'after-execute', + operations: infos, status: buildStatus }; sendWebSocketMessage(afterExecuteMessage); @@ -435,3 +487,7 @@ function getRepositoryIdentifier(rushConfiguration: RushConfiguration): string { return `${os.hostname()} - ${rushConfiguration.rushJsonFolder}`; } + +function getLogServePathForProject(logServePath: string, packageName: string): string { + return `${logServePath}/${packageName}`; +} diff --git a/rush-plugins/rush-serve-plugin/src/schemas/rush-serve-plugin-options.schema.json b/rush-plugins/rush-serve-plugin/src/schemas/rush-serve-plugin-options.schema.json index f0184060284..f57cb6a7ba6 100644 --- a/rush-plugins/rush-serve-plugin/src/schemas/rush-serve-plugin-options.schema.json +++ b/rush-plugins/rush-serve-plugin/src/schemas/rush-serve-plugin-options.schema.json @@ -33,6 +33,12 @@ "pattern": "^/(?:[a-zA-Z0-9_$-]+(?:/[a-zA-Z0-9_$-]+)*)?$" }, + "logServePath": { + "type": "string", + "description": "The URL path at which to host Rush log files. If not specified, log files will not be served.", + "pattern": "^/(?:[a-zA-Z0-9_$-]+(?:/[a-zA-Z0-9_$-]+)*)?$" + }, + "globalRouting": { "type": "array", "description": "Routing rules for files that are associated with the entire workspace, rather than a single project (e.g. files output by Rush plugins).",