From 5ff2a62c60505eb0108c3ac57eefe7b7cb236031 Mon Sep 17 00:00:00 2001 From: David Michon Date: Tue, 24 Sep 2024 23:54:55 +0000 Subject: [PATCH 1/6] [rush] Split ProjectChangeAnalyzer, fix build cache hashes --- ...ject-change-analyzer_2024-01-09-22-06.json | 10 + ...ject-change-analyzer_2024-01-09-22-11.json | 10 + ...ject-change-analyzer_2024-09-24-23-54.json | 10 + common/reviews/api/lookup-by-path.api.md | 10 +- common/reviews/api/rush-lib.api.md | 34 +- libraries/lookup-by-path/src/LookupByPath.ts | 109 ++-- libraries/lookup-by-path/src/index.ts | 2 +- .../cli/scriptActions/PhasedScriptAction.ts | 51 +- .../cli/test/RushCommandLineParser.test.ts | 8 +- libraries/rush-lib/src/index.ts | 11 +- .../src/logic/ProjectChangeAnalyzer.ts | 492 +++++++++--------- .../rush-lib/src/logic/ProjectWatcher.ts | 56 +- .../src/logic/buildCache/ProjectBuildCache.ts | 130 ++--- .../buildCache/getHashesForGlobsAsync.ts | 104 ---- .../buildCache/test/ProjectBuildCache.test.ts | 23 +- .../src/logic/operations/BuildPlanPlugin.ts | 13 +- .../operations/CacheableOperationPlugin.ts | 331 ++++++------ .../src/logic/operations/LegacySkipPlugin.ts | 22 +- .../operations/test/BuildPlanPlugin.test.ts | 15 +- .../src/logic/snapshots/InputSnapshot.ts | 455 ++++++++++++++++ .../snapshots/test/InputSnapshot.test.ts | 469 +++++++++++++++++ .../__snapshots__/InputSnapshot.test.ts.snap | 52 ++ .../logic/test/ProjectChangeAnalyzer.test.ts | 391 +++----------- .../ProjectChangeAnalyzer.test.ts.snap | 3 - .../src/pluginFramework/PhasedCommandHooks.ts | 10 +- 25 files changed, 1740 insertions(+), 1081 deletions(-) create mode 100644 common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-06.json create mode 100644 common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-11.json create mode 100644 common/changes/@rushstack/lookup-by-path/split-project-change-analyzer_2024-09-24-23-54.json delete mode 100644 libraries/rush-lib/src/logic/buildCache/getHashesForGlobsAsync.ts create mode 100644 libraries/rush-lib/src/logic/snapshots/InputSnapshot.ts create mode 100644 libraries/rush-lib/src/logic/snapshots/test/InputSnapshot.test.ts create mode 100644 libraries/rush-lib/src/logic/snapshots/test/__snapshots__/InputSnapshot.test.ts.snap delete mode 100644 libraries/rush-lib/src/logic/test/__snapshots__/ProjectChangeAnalyzer.test.ts.snap diff --git a/common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-06.json b/common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-06.json new file mode 100644 index 00000000000..51d6f1403a2 --- /dev/null +++ b/common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-06.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "[CACHE BREAK] Alter the computation of build cache IDs to depend on the graph of operations in the build and therefore account for multiple phases, rather than only the declared dependencies. Ensure that `dependsOnEnvVars` and command line parameters that affect upstream phases impact the cache IDs of downstream operations.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-11.json b/common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-11.json new file mode 100644 index 00000000000..ff408bd0f32 --- /dev/null +++ b/common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-11.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "(BREAKING CHANGE) Replace use of `ProjectChangeAnalyzer` in phased command hooks with a new `InputSnapshot` data structure that is completely synchronous and does not perform any disk operations. Perform all disk operations and state computation prior to executing the build graph.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/changes/@rushstack/lookup-by-path/split-project-change-analyzer_2024-09-24-23-54.json b/common/changes/@rushstack/lookup-by-path/split-project-change-analyzer_2024-09-24-23-54.json new file mode 100644 index 00000000000..3c1d5f9f7e5 --- /dev/null +++ b/common/changes/@rushstack/lookup-by-path/split-project-change-analyzer_2024-09-24-23-54.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/lookup-by-path", + "comment": "Add `IReadonlyLookupByPath` interface to help unit tests for functions that consume `LookupByPath`.", + "type": "minor" + } + ], + "packageName": "@rushstack/lookup-by-path" +} \ No newline at end of file diff --git a/common/reviews/api/lookup-by-path.api.md b/common/reviews/api/lookup-by-path.api.md index 1d779056878..6fcf473ecd5 100644 --- a/common/reviews/api/lookup-by-path.api.md +++ b/common/reviews/api/lookup-by-path.api.md @@ -12,7 +12,15 @@ export interface IPrefixMatch { } // @beta -export class LookupByPath { +export interface IReadonlyLookupByPath { + findChildPath(childPath: string): TItem | undefined; + findChildPathFromSegments(childPathSegments: Iterable): TItem | undefined; + findLongestPrefixMatch(query: string): IPrefixMatch | undefined; + groupByChild(infoByPath: Map): Map>; +} + +// @beta +export class LookupByPath implements IReadonlyLookupByPath { constructor(entries?: Iterable<[string, TItem]>, delimiter?: string); readonly delimiter: string; findChildPath(childPath: string): TItem | undefined; diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 005423e2727..154718c5471 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -462,7 +462,7 @@ export interface IEnvironmentConfigurationInitializeOptions { // @alpha export interface IExecuteOperationsContext extends ICreateOperationsContext { - readonly projectChangeAnalyzer: ProjectChangeAnalyzer; + readonly inputSnapshot?: IInputSnapshot; } // @alpha @@ -521,6 +521,19 @@ export interface IGetChangedProjectsOptions { export interface IGlobalCommand extends IRushCommand { } +// @beta +export interface IInputSnapshot { + getOperationOwnStateHash(project: IRushConfigurationProjectForSnapshot, operationName?: string): string; + getTrackedFileHashesForOperation(project: IRushConfigurationProjectForSnapshot, operationName?: string): ReadonlyMap; + readonly hashes: ReadonlyMap; + readonly rootDirectory: string; +} + +// @beta +export interface IInputSnapshotProvider { + (): Promise; +} + // @public export interface ILaunchOptions { alreadyReportedNodeTooNewError?: boolean; @@ -759,16 +772,6 @@ export interface IPnpmPeerDependencyRules { export { IPrefixMatch } -// @internal (undocumented) -export interface _IRawRepoState { - // (undocumented) - projectState: Map> | undefined; - // (undocumented) - rawHashes: Map; - // (undocumented) - rootDir: string; -} - // @beta export interface IRushCommand { readonly actionName: string; @@ -798,6 +801,9 @@ export interface IRushCommandLineSpec { actions: IRushCommandLineAction[]; } +// @beta (undocumented) +export type IRushConfigurationProjectForSnapshot = Pick; + // @alpha (undocumented) export interface IRushPhaseSharding { count: number; @@ -1113,16 +1119,12 @@ export type PnpmStoreOptions = PnpmStoreLocation; export class ProjectChangeAnalyzer { constructor(rushConfiguration: RushConfiguration); // @internal (undocumented) - _ensureInitializedAsync(terminal: ITerminal): Promise<_IRawRepoState | undefined>; - // (undocumented) _filterProjectDataAsync(project: RushConfigurationProject, unfilteredProjectData: Map, rootDir: string, terminal: ITerminal): Promise>; getChangedProjectsAsync(options: IGetChangedProjectsOptions): Promise>; // (undocumented) protected getChangesByProject(lookup: LookupByPath, changedFiles: Map): Map>; // @internal - _tryGetProjectDependenciesAsync(project: RushConfigurationProject, terminal: ITerminal): Promise | undefined>; - // @internal - _tryGetProjectStateHashAsync(project: RushConfigurationProject, terminal: ITerminal): Promise; + _tryGetSnapshotProviderAsync(projectConfigurations: ReadonlyMap, terminal: ITerminal): Promise; } // @public diff --git a/libraries/lookup-by-path/src/LookupByPath.ts b/libraries/lookup-by-path/src/LookupByPath.ts index 1f89630af01..c4b7d01366d 100644 --- a/libraries/lookup-by-path/src/LookupByPath.ts +++ b/libraries/lookup-by-path/src/LookupByPath.ts @@ -46,6 +46,69 @@ export interface IPrefixMatch { lastMatch?: IPrefixMatch; } +/** + * The readonly component of `LookupByPath`, to simplify unit testing. + * + * @beta + */ +export interface IReadonlyLookupByPath { + /** + * Searches for the item associated with `childPath`, or the nearest ancestor of that path that + * has an associated item. + * + * @returns the found item, or `undefined` if no item was found + * + * @example + * ```ts + * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]); + * trie.findChildPath('foo/baz'); // returns 1 + * trie.findChildPath('foo/bar/baz'); // returns 2 + * ``` + */ + findChildPath(childPath: string): TItem | undefined; + + /** + * Searches for the item for which the recorded prefix is the longest matching prefix of `query`. + * Obtains both the item and the length of the matched prefix, so that the remainder of the path can be + * extracted. + * + * @returns the found item and the length of the matched prefix, or `undefined` if no item was found + * + * @example + * ```ts + * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]); + * trie.findLongestPrefixMatch('foo/baz'); // returns { item: 1, index: 3 } + * trie.findLongestPrefixMatch('foo/bar/baz'); // returns { item: 2, index: 7 } + * ``` + */ + findLongestPrefixMatch(query: string): IPrefixMatch | undefined; + + /** + * Searches for the item associated with `childPathSegments`, or the nearest ancestor of that path that + * has an associated item. + * + * @returns the found item, or `undefined` if no item was found + * + * @example + * ```ts + * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]); + * trie.findChildPathFromSegments(['foo', 'baz']); // returns 1 + * trie.findChildPathFromSegments(['foo','bar', 'baz']); // returns 2 + * ``` + */ + findChildPathFromSegments(childPathSegments: Iterable): TItem | undefined; + + /** + * Groups the provided map of info by the nearest entry in the trie that contains the path. If the path + * is not found in the trie, the info is ignored. + * + * @returns The grouped info, grouped by the nearest entry in the trie that contains the path + * + * @param infoByPath - The info to be grouped, keyed by path + */ + groupByChild(infoByPath: Map): Map>; +} + /** * This class is used to associate path-like-strings, such as those returned by `git` commands, * with entities that correspond with ancestor folders, such as Rush Projects or npm packages. @@ -66,7 +129,7 @@ export interface IPrefixMatch { * ``` * @beta */ -export class LookupByPath { +export class LookupByPath implements IReadonlyLookupByPath { /** * The delimiter used to split paths */ @@ -178,52 +241,21 @@ export class LookupByPath { } /** - * Searches for the item associated with `childPath`, or the nearest ancestor of that path that - * has an associated item. - * - * @returns the found item, or `undefined` if no item was found - * - * @example - * ```ts - * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]); - * trie.findChildPath('foo/baz'); // returns 1 - * trie.findChildPath('foo/bar/baz'); // returns 2 - * ``` + * {@inheritdoc IReadonlyLookupByPath} */ public findChildPath(childPath: string): TItem | undefined { return this.findChildPathFromSegments(LookupByPath.iteratePathSegments(childPath, this.delimiter)); } /** - * Searches for the item for which the recorded prefix is the longest matching prefix of `query`. - * Obtains both the item and the length of the matched prefix, so that the remainder of the path can be - * extracted. - * - * @returns the found item and the length of the matched prefix, or `undefined` if no item was found - * - * @example - * ```ts - * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]); - * trie.findLongestPrefixMatch('foo/baz'); // returns { item: 1, index: 3 } - * trie.findLongestPrefixMatch('foo/bar/baz'); // returns { item: 2, index: 7 } - * ``` + * {@inheritdoc IReadonlyLookupByPath} */ public findLongestPrefixMatch(query: string): IPrefixMatch | undefined { return this._findLongestPrefixMatch(LookupByPath._iteratePrefixes(query, this.delimiter)); } /** - * Searches for the item associated with `childPathSegments`, or the nearest ancestor of that path that - * has an associated item. - * - * @returns the found item, or `undefined` if no item was found - * - * @example - * ```ts - * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]); - * trie.findChildPathFromSegments(['foo', 'baz']); // returns 1 - * trie.findChildPathFromSegments(['foo','bar', 'baz']); // returns 2 - * ``` + * {@inheritdoc IReadonlyLookupByPath} */ public findChildPathFromSegments(childPathSegments: Iterable): TItem | undefined { let node: IPathTrieNode = this._root; @@ -247,12 +279,7 @@ export class LookupByPath { } /** - * Groups the provided map of info by the nearest entry in the trie that contains the path. If the path - * is not found in the trie, the info is ignored. - * - * @returns The grouped info, grouped by the nearest entry in the trie that contains the path - * - * @param infoByPath - The info to be grouped, keyed by path + * {@inheritdoc IReadonlyLookupByPath} */ public groupByChild(infoByPath: Map): Map> { const groupedInfoByChild: Map> = new Map(); diff --git a/libraries/lookup-by-path/src/index.ts b/libraries/lookup-by-path/src/index.ts index 7cc91e2647e..1beccb0d9ec 100644 --- a/libraries/lookup-by-path/src/index.ts +++ b/libraries/lookup-by-path/src/index.ts @@ -7,5 +7,5 @@ * @packageDocumentation */ -export type { IPrefixMatch } from './LookupByPath'; +export type { IPrefixMatch, IReadonlyLookupByPath } from './LookupByPath'; export { LookupByPath } from './LookupByPath'; diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index e37701e73a0..6e4d5fe311b 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -44,6 +44,7 @@ import type { ITelemetryData, ITelemetryOperationResult } from '../../logic/Tele import { parseParallelism } from '../parsing/ParseParallelism'; import { CobuildConfiguration } from '../../api/CobuildConfiguration'; import { CacheableOperationPlugin } from '../../logic/operations/CacheableOperationPlugin'; +import type { IInputSnapshot, IInputSnapshotProvider } from '../../logic/snapshots/InputSnapshot'; import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { LegacySkipPlugin } from '../../logic/operations/LegacySkipPlugin'; import { ValidateOperationsPlugin } from '../../logic/operations/ValidateOperationsPlugin'; @@ -80,7 +81,8 @@ interface IInitialRunPhasesOptions { } interface IRunPhasesOptions extends IInitialRunPhasesOptions { - initialState: ProjectChangeAnalyzer; + snapshotProvider: IInputSnapshotProvider | undefined; + initialSnapshot: IInputSnapshot | undefined; executionManagerOptions: IOperationExecutionManagerOptions; } @@ -536,24 +538,34 @@ export class PhasedScriptAction extends BaseScriptAction { terminal } = options; + const { projectConfigurations } = initialCreateOperationsContext; + const operations: Set = await this.hooks.createOperations.promise( new Set(), initialCreateOperationsContext ); - const projectChangeAnalyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); - terminal.write('Analyzing repo state... '); const repoStateStopwatch: Stopwatch = new Stopwatch(); repoStateStopwatch.start(); - await projectChangeAnalyzer._ensureInitializedAsync(terminal); + + const analyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); + const snapshotProvider: IInputSnapshotProvider | undefined = + await analyzer._tryGetSnapshotProviderAsync(projectConfigurations, terminal); + const initialSnapshot: IInputSnapshot | undefined = await snapshotProvider?.(); + repoStateStopwatch.stop(); terminal.writeLine(`DONE (${repoStateStopwatch.toString()})`); + if (!initialSnapshot) { + terminal.writeLine( + `The Rush monorepo is not in a Git repository. Rush will proceed without incremental build support.` + ); + } terminal.writeLine(); const initialExecuteOperationsContext: IExecuteOperationsContext = { ...initialCreateOperationsContext, - projectChangeAnalyzer + inputSnapshot: initialSnapshot }; const executionManagerOptions: IOperationExecutionManagerOptions = { @@ -577,7 +589,8 @@ export class PhasedScriptAction extends BaseScriptAction { return { ...options, executionManagerOptions, - initialState: projectChangeAnalyzer + snapshotProvider, + initialSnapshot }; } @@ -652,26 +665,40 @@ export class PhasedScriptAction extends BaseScriptAction { * 3) Goto (1) */ private async _runWatchPhasesAsync(options: IRunPhasesOptions): Promise { - const { initialState, initialCreateOperationsContext, executionManagerOptions, stopwatch, terminal } = - options; + const { + snapshotProvider, + initialSnapshot: initialState, + initialCreateOperationsContext, + executionManagerOptions, + stopwatch, + terminal + } = options; const phaseOriginal: Set = new Set(this._watchPhases); const phaseSelection: Set = new Set(this._watchPhases); const { projectSelection: projectsToWatch } = initialCreateOperationsContext; + if (!snapshotProvider || !initialState) { + terminal.writeErrorLine( + `Cannot watch for changes if the Rush repo is not in a Git repository, exiting.` + ); + throw new AlreadyReportedError(); + } + // Use async import so that we don't pay the cost for sync builds const { ProjectWatcher } = await import( /* webpackChunkName: 'ProjectWatcher' */ '../../logic/ProjectWatcher' ); - const projectWatcher: ProjectWatcher = new ProjectWatcher({ + const projectWatcher: typeof ProjectWatcher.prototype = new ProjectWatcher({ + snapshotProvider, + initialState, debounceMs: this._watchDebounceMs, rushConfiguration: this.rushConfiguration, projectsToWatch, - terminal, - initialState + terminal }); // Ensure process.stdin allows interactivity before using TTY-only APIs @@ -724,7 +751,7 @@ export class PhasedScriptAction extends BaseScriptAction { const executeOperationsContext: IExecuteOperationsContext = { ...initialCreateOperationsContext, isInitial: false, - projectChangeAnalyzer: state, + inputSnapshot: state, projectsInUnknownState: changedProjects, phaseOriginal, phaseSelection, diff --git a/libraries/rush-lib/src/cli/test/RushCommandLineParser.test.ts b/libraries/rush-lib/src/cli/test/RushCommandLineParser.test.ts index b14510c8dee..6316c8fe6ce 100644 --- a/libraries/rush-lib/src/cli/test/RushCommandLineParser.test.ts +++ b/libraries/rush-lib/src/cli/test/RushCommandLineParser.test.ts @@ -7,10 +7,16 @@ jest.mock(`@rushstack/package-deps-hash`, () => { return dir; }, getRepoStateAsync(): ReadonlyMap { - return new Map(); + return new Map([['common/config/rush/npm-shrinkwrap.json', 'hash']]); }, getRepoChangesAsync(): ReadonlyMap { return new Map(); + }, + getGitHashForFiles(filePaths: Iterable): ReadonlyMap { + return new Map(Array.from(filePaths, (filePath: string) => [filePath, filePath])); + }, + hashFilesAsync(rootDirectory: string, filePaths: Iterable): ReadonlyMap { + return new Map(Array.from(filePaths, (filePath: string) => [filePath, filePath])); } }; }); diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index f7d2e9a2278..fb7c1c35a42 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -121,11 +121,12 @@ export { CustomTipType } from './api/CustomTipsConfiguration'; -export { - ProjectChangeAnalyzer, - type IGetChangedProjectsOptions, - type IRawRepoState as _IRawRepoState -} from './logic/ProjectChangeAnalyzer'; +export { ProjectChangeAnalyzer, type IGetChangedProjectsOptions } from './logic/ProjectChangeAnalyzer'; +export type { + IInputSnapshot, + IInputSnapshotProvider, + IRushConfigurationProjectForSnapshot +} from './logic/snapshots/InputSnapshot'; export type { IOperationRunner, IOperationRunnerContext } from './logic/operations/IOperationRunner'; export type { diff --git a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts index acda3b3ccd8..fa739d22c12 100644 --- a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts +++ b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts @@ -2,27 +2,31 @@ // See LICENSE in the project root for license information. import * as path from 'path'; -import * as crypto from 'crypto'; import ignore, { type Ignore } from 'ignore'; +import type { IReadonlyLookupByPath, LookupByPath } from '@rushstack/lookup-by-path'; +import { Path, FileSystem, Async, AlreadyReportedError } from '@rushstack/node-core-library'; import { getRepoChanges, getRepoRoot, getRepoStateAsync, + hashFilesAsync, type IFileDiffStatus } from '@rushstack/package-deps-hash'; -import { Path, FileSystem, Async } from '@rushstack/node-core-library'; import type { ITerminal } from '@rushstack/terminal'; import type { RushConfiguration } from '../api/RushConfiguration'; import { RushProjectConfiguration } from '../api/RushProjectConfiguration'; -import { Git } from './Git'; -import { BaseProjectShrinkwrapFile } from './base/BaseProjectShrinkwrapFile'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; -import { RushConstants } from './RushConstants'; -import type { LookupByPath } from '@rushstack/lookup-by-path'; +import { BaseProjectShrinkwrapFile } from './base/BaseProjectShrinkwrapFile'; import { PnpmShrinkwrapFile } from './pnpm/PnpmShrinkwrapFile'; -import { UNINITIALIZED } from '../utilities/Utilities'; +import { Git } from './Git'; +import { + type IRushSnapshotProjectMetadata, + type IInputSnapshot, + InputSnapshot, + type IInputSnapshotProvider +} from './snapshots/InputSnapshot'; /** * @beta @@ -46,12 +50,6 @@ export interface IGetChangedProjectsOptions { enableFiltering: boolean; } -interface IGitState { - gitPath: string; - hashes: Map; - rootDir: string; -} - /** * @internal */ @@ -65,13 +63,6 @@ export interface IRawRepoState { * @beta */ export class ProjectChangeAnalyzer { - /** - * UNINITIALIZED === we haven't looked - * undefined === data isn't available (i.e. - git isn't present) - */ - private _data: IRawRepoState | UNINITIALIZED | undefined = UNINITIALIZED; - private readonly _filteredData: Map> = new Map(); - private readonly _projectStateCache: Map = new Map(); private readonly _rushConfiguration: RushConfiguration; private readonly _git: Git; @@ -80,134 +71,6 @@ export class ProjectChangeAnalyzer { this._git = new Git(this._rushConfiguration); } - /** - * Try to get a list of the specified project's dependencies and their hashes. - * - * @remarks - * If the data can't be generated (i.e. - if Git is not present) this returns undefined. - * - * @internal - */ - public async _tryGetProjectDependenciesAsync( - project: RushConfigurationProject, - terminal: ITerminal - ): Promise | undefined> { - // Check the cache for any existing data - let filteredProjectData: Map | undefined = this._filteredData.get(project); - if (filteredProjectData) { - return filteredProjectData; - } - - const data: IRawRepoState | undefined = await this._ensureInitializedAsync(terminal); - - if (!data) { - return undefined; - } - - const { projectState, rootDir } = data; - - if (projectState === undefined) { - return undefined; - } - - const unfilteredProjectData: Map | undefined = projectState.get(project); - if (!unfilteredProjectData) { - throw new Error(`Project "${project.packageName}" does not exist in the current Rush configuration.`); - } - - filteredProjectData = await this._filterProjectDataAsync( - project, - unfilteredProjectData, - rootDir, - terminal - ); - - this._filteredData.set(project, filteredProjectData); - return filteredProjectData; - } - - /** - * @internal - */ - public async _ensureInitializedAsync(terminal: ITerminal): Promise { - if (this._data === UNINITIALIZED) { - this._data = await this._getDataAsync(terminal); - } - - return this._data; - } - - /** - * The project state hash is calculated in the following way: - * - Project dependencies are collected (see ProjectChangeAnalyzer.getPackageDeps) - * - If project dependencies cannot be collected (i.e. - if Git isn't available), - * this function returns `undefined` - * - The (path separator normalized) repo-root-relative dependencies' file paths are sorted - * - A SHA1 hash is created and each (sorted) file path is fed into the hash and then its - * Git SHA is fed into the hash - * - A hex digest of the hash is returned - * - * @internal - */ - public async _tryGetProjectStateHashAsync( - project: RushConfigurationProject, - terminal: ITerminal - ): Promise { - let projectState: string | undefined = this._projectStateCache.get(project); - if (!projectState) { - const packageDeps: Map | undefined = await this._tryGetProjectDependenciesAsync( - project, - terminal - ); - - if (!packageDeps) { - return undefined; - } else { - const sortedPackageDepsFiles: string[] = Array.from(packageDeps.keys()).sort(); - const hash: crypto.Hash = crypto.createHash('sha1'); - for (const packageDepsFile of sortedPackageDepsFiles) { - hash.update(packageDepsFile); - hash.update(RushConstants.hashDelimiter); - hash.update(packageDeps.get(packageDepsFile)!); - hash.update(RushConstants.hashDelimiter); - } - - projectState = hash.digest('hex'); - this._projectStateCache.set(project, projectState); - } - } - - return projectState; - } - - public async _filterProjectDataAsync( - project: RushConfigurationProject, - unfilteredProjectData: Map, - rootDir: string, - terminal: ITerminal - ): Promise> { - const ignoreMatcher: Ignore | undefined = await this._getIgnoreMatcherForProjectAsync(project, terminal); - if (!ignoreMatcher) { - return unfilteredProjectData; - } - - const projectKey: string = path.relative(rootDir, project.projectFolder); - const projectKeyLength: number = projectKey.length + 1; - - // At this point, `filePath` is guaranteed to start with `projectKey`, so - // we can safely slice off the first N characters to get the file path relative to the - // root of the project. - const filteredProjectData: Map = new Map(); - for (const [filePath, value] of unfilteredProjectData) { - const relativePath: string = filePath.slice(projectKeyLength); - if (!ignoreMatcher.ignores(relativePath)) { - // Add the file path to the filtered data if it is not ignored - filteredProjectData.set(filePath, value); - } - } - return filteredProjectData; - } - /** * Gets a list of projects that have changed in the current state of the repo * when compared to the specified branch, optionally taking the shrinkwrap and settings in @@ -331,66 +194,187 @@ export class ProjectChangeAnalyzer { return lookup.groupByChild(changedFiles); } - private async _getDataAsync(terminal: ITerminal): Promise { - const repoState: IGitState | undefined = await this._getRepoDepsAsync(terminal); - if (!repoState) { - // Mark as resolved, but no data - return { - projectState: undefined, - rootDir: this._rushConfiguration.rushJsonFolder, - rawHashes: new Map() - }; - } + /** + * Gets a snapshot of the input state of the Rush workspace that can be queried for incremental + * build operations and use by the build cache. + * @internal + */ + public async _tryGetSnapshotProviderAsync( + projectConfigurations: ReadonlyMap, + terminal: ITerminal + ): Promise { + try { + const gitPath: string = this._git.getGitPathOrThrow(); + + if (!this._git.isPathUnderGitWorkingTree()) { + return; + } - const lookup: LookupByPath = this._rushConfiguration.getProjectLookupForRoot( - repoState.rootDir - ); - const projectHashDeps: Map> = new Map(); + const rushConfiguration: RushConfiguration = this._rushConfiguration; - for (const project of this._rushConfiguration.projects) { - projectHashDeps.set(project, new Map()); - } + // Do not use getGitInfo().root; it is the root of the *primary* worktree, not the *current* one. + const rootDirectory: string = getRepoRoot(rushConfiguration.rushJsonFolder, gitPath); - const { hashes: repoDeps, rootDir } = repoState; - - // Currently, only pnpm handles project shrinkwraps - if (this._rushConfiguration.packageManager !== 'pnpm') { - const currentVariant: string | undefined = - await this._rushConfiguration.getCurrentlyInstalledVariantAsync(); - // Add the shrinkwrap file to every project's dependencies - const shrinkwrapFile: string = Path.convertToSlashes( - path.relative( - rootDir, - this._rushConfiguration.defaultSubspace.getCommittedShrinkwrapFilePath(currentVariant) - ) - ); + // Load the rush-project.json files for the whole repository + const additionalGlobs: IAdditionalGlob[] = []; - const shrinkwrapHash: string | undefined = repoDeps.get(shrinkwrapFile); + const projectMap: Map = new Map(); - for (const projectDeps of projectHashDeps.values()) { - if (shrinkwrapHash) { - projectDeps.set(shrinkwrapFile, shrinkwrapHash); + for (const project of rushConfiguration.projects) { + const projectConfig: RushProjectConfiguration | undefined = projectConfigurations.get(project); + + const additionalFilesByOperationName: Map> = new Map(); + const projectMetadata: IRushSnapshotProjectMetadata = { + projectConfig, + additionalFilesByOperationName + }; + projectMap.set(project, projectMetadata); + + if (projectConfig) { + const { operationSettingsByOperationName } = projectConfig; + for (const [operationName, { dependsOnAdditionalFiles }] of operationSettingsByOperationName) { + if (dependsOnAdditionalFiles) { + const additionalFilesForOperation: Set = new Set(); + additionalFilesByOperationName.set(operationName, additionalFilesForOperation); + for (const pattern of dependsOnAdditionalFiles) { + additionalGlobs.push({ + project, + operationName, + additionalFilesForOperation, + pattern + }); + } + } + } } } - } - // Sort each project folder into its own package deps hash - for (const [filePath, fileHash] of repoDeps) { - // lookups in findChildPath are O(K) - // K being the maximum folder depth of any project in rush.json (usually on the order of 3) - const owningProject: RushConfigurationProject | undefined = lookup.findChildPath(filePath); + // Include project shrinkwrap files as part of the computation + const additionalRelativePathsToHash: string[] = []; + const globalAdditionalFiles: string[] = []; + if (rushConfiguration.packageManager === 'pnpm') { + const absoluteFilePathsToCheck: string[] = []; + + for (const project of rushConfiguration.projects) { + const projectShrinkwrapFilePath: string = BaseProjectShrinkwrapFile.getFilePathForProject(project); + absoluteFilePathsToCheck.push(projectShrinkwrapFilePath); + const relativeProjectShrinkwrapFilePath: string = Path.convertToSlashes( + path.relative(rootDirectory, projectShrinkwrapFilePath) + ); + + additionalRelativePathsToHash.push(relativeProjectShrinkwrapFilePath); + } - if (owningProject) { - const owningProjectHashDeps: Map = projectHashDeps.get(owningProject)!; - owningProjectHashDeps.set(filePath, fileHash); + await Async.forEachAsync(absoluteFilePathsToCheck, async (filePath: string) => { + if (!rushConfiguration.subspacesFeatureEnabled && !(await FileSystem.existsAsync(filePath))) { + throw new Error( + `A project dependency file (${filePath}) is missing. You may need to run ` + + '"rush install" or "rush update".' + ); + } + }); + } else { + // Add the shrinkwrap file to every project's dependencies + const currentVariant: string | undefined = + await this._rushConfiguration.getCurrentlyInstalledVariantAsync(); + + const shrinkwrapFile: string = Path.convertToSlashes( + path.relative( + rootDirectory, + rushConfiguration.defaultSubspace.getCommittedShrinkwrapFilePath(currentVariant) + ) + ); + + globalAdditionalFiles.push(shrinkwrapFile); } + + const lookupByPath: IReadonlyLookupByPath = + this._rushConfiguration.getProjectLookupForRoot(rootDirectory); + + return async function tryGetSnapshotAsync(): Promise { + try { + const [hashes, additionalFiles] = await Promise.all([ + getRepoStateAsync(rootDirectory, additionalRelativePathsToHash, gitPath), + getAdditionalFilesFromRushProjectConfigurationAsync( + additionalGlobs, + lookupByPath, + rootDirectory, + terminal + ) + ]); + + for (const file of additionalFiles) { + if (hashes.has(file)) { + additionalFiles.delete(file); + } + } + + const additionalHashes: Map = new Map( + await hashFilesAsync(rootDirectory, additionalFiles, gitPath) + ); + + return new InputSnapshot({ + additionalHashes, + globalAdditionalFiles, + hashes, + lookupByPath, + projectMap: projectMap, + rootDir: rootDirectory + }); + } catch (e) { + // If getRepoState fails, don't fail the whole build. Treat this case as if we don't know anything about + // the state of the files in the repo. This can happen if the environment doesn't have Git. + terminal.writeWarningLine( + `Error calculating the state of the repo. (inner error: ${ + e.stack ?? e.message ?? e + }). Continuing without diffing files.` + ); + + return; + } + }; + } catch (e) { + // If getRepoState fails, don't fail the whole build. Treat this case as if we don't know anything about + // the state of the files in the repo. This can happen if the environment doesn't have Git. + terminal.writeWarningLine( + `Error calculating the state of the repo. (inner error: ${ + e.stack ?? e.message ?? e + }). Continuing without diffing files.` + ); + + return; } + } - return { - projectState: projectHashDeps, - rootDir, - rawHashes: repoState.hashes - }; + /** + * @internal + */ + public async _filterProjectDataAsync( + project: RushConfigurationProject, + unfilteredProjectData: Map, + rootDir: string, + terminal: ITerminal + ): Promise> { + const ignoreMatcher: Ignore | undefined = await this._getIgnoreMatcherForProjectAsync(project, terminal); + if (!ignoreMatcher) { + return unfilteredProjectData; + } + + const projectKey: string = path.relative(rootDir, project.projectFolder); + const projectKeyLength: number = projectKey.length + 1; + + // At this point, `filePath` is guaranteed to start with `projectKey`, so + // we can safely slice off the first N characters to get the file path relative to the + // root of the project. + const filteredProjectData: Map = new Map(); + for (const [filePath, value] of unfilteredProjectData) { + const relativePath: string = filePath.slice(projectKeyLength); + if (!ignoreMatcher.ignores(relativePath)) { + // Add the file path to the filtered data if it is not ignored + filteredProjectData.set(filePath, value); + } + } + return filteredProjectData; } private async _getIgnoreMatcherForProjectAsync( @@ -406,59 +390,77 @@ export class ProjectChangeAnalyzer { return ignoreMatcher; } } +} + +interface IAdditionalGlob { + project: RushConfigurationProject; + operationName: string; + additionalFilesForOperation: Set; + pattern: string; +} + +async function getAdditionalFilesFromRushProjectConfigurationAsync( + additionalGlobs: IAdditionalGlob[], + rootRelativeLookupByPath: IReadonlyLookupByPath, + rootDirectory: string, + terminal: ITerminal +): Promise> { + const additionalFilesFromRushProjectConfiguration: Set = new Set(); + + if (!additionalGlobs.length) { + return additionalFilesFromRushProjectConfiguration; + } - private async _getRepoDepsAsync(terminal: ITerminal): Promise { - try { - const gitPath: string = this._git.getGitPathOrThrow(); + const { default: glob } = await import('fast-glob'); + await Async.forEachAsync(additionalGlobs, async (item: IAdditionalGlob) => { + const { project, operationName, additionalFilesForOperation, pattern } = item; + const matches: string[] = await glob(pattern, { + cwd: project.projectFolder, + onlyFiles: true, + // We want to keep path's type unchanged, + // i.e. if the pattern was a relative path, then matched paths should also be relative paths + // if the pattern was an absolute path, then matched paths should also be absolute paths + // + // We are doing this because these paths are going to be used to calculate operation state hashes and some users + // might choose to depend on global files (e.g. `/etc/os-release`) and some might choose to depend on local non-project files + // (e.g. `../path/to/workspace/file`) + // + // In both cases we want that path to the resource to be the same on all machines, + // regardless of what is the current working directory. + // + // That being said, we want to keep `absolute` options here as false: + absolute: false + }); + + for (const match of matches) { + // The glob result is relative to the project folder, but we want it to be relative to the repo root + const rootRelativeFilePath: string = Path.convertToSlashes( + path.relative(rootDirectory, path.resolve(project.projectFolder, match)) + ); - if (this._git.isPathUnderGitWorkingTree()) { - // Do not use getGitInfo().root; it is the root of the *primary* worktree, not the *current* one. - const rootDir: string = getRepoRoot(this._rushConfiguration.rushJsonFolder, gitPath); - // Load the package deps hash for the whole repository - // Include project shrinkwrap files as part of the computation - const additionalFilesToHash: string[] = []; - - if (this._rushConfiguration.packageManager === 'pnpm') { - await Async.forEachAsync( - this._rushConfiguration.projects, - async (project: RushConfigurationProject) => { - const projectShrinkwrapFilePath: string = - BaseProjectShrinkwrapFile.getFilePathForProject(project); - if (!(await FileSystem.existsAsync(projectShrinkwrapFilePath))) { - // Missing shrinkwrap of subspace project is allowed because subspace projects can be partial installed - if (this._rushConfiguration.subspacesFeatureEnabled) { - return; - } - throw new Error( - `A project dependency file (${projectShrinkwrapFilePath}) is missing. You may need to run ` + - '"rush install" or "rush update".' - ); - } - const relativeProjectShrinkwrapFilePath: string = Path.convertToSlashes( - path.relative(rootDir, projectShrinkwrapFilePath) - ); - additionalFilesToHash.push(relativeProjectShrinkwrapFilePath); - } + if (rootRelativeFilePath.startsWith('../')) { + // The target file is outside of the Git tree, use the original result of the match. + additionalFilesFromRushProjectConfiguration.add(match); + additionalFilesForOperation.add(match); + } else { + // The target file is inside of the Git tree, find out if it is in a Rush project. + const projectMatch: RushConfigurationProject | undefined = + rootRelativeLookupByPath.findChildPath(rootRelativeFilePath); + if (projectMatch && projectMatch !== project) { + terminal.writeErrorLine( + `In project "${project.packageName}" ("${project.projectRelativeFolder}"), ` + + `config for operation "${operationName}" specifies a glob "${pattern}" that selects a file "${rootRelativeFilePath}" in a different workspace project ` + + `"${projectMatch.packageName}" ("${projectMatch.projectRelativeFolder}"). ` + + `This is forbidden. The "dependsOnAdditionalFiles" property of "rush-project.json" may only be used to refer to non-workspace files, non-project files, ` + + `or untracked files in the current project. To depend on files in another workspace project, use "devDependencies" in "package.json".` ); + throw new AlreadyReportedError(); } - - const hashes: Map = await getRepoStateAsync(rootDir, additionalFilesToHash, gitPath); - return { - gitPath, - hashes, - rootDir - }; - } else { - return undefined; + additionalFilesForOperation.add(rootRelativeFilePath); + additionalFilesFromRushProjectConfiguration.add(rootRelativeFilePath); } - } catch (e) { - // If getPackageDeps fails, don't fail the whole build. Treat this case as if we don't know anything about - // the state of the files in the repo. This can happen if the environment doesn't have Git. - terminal.writeWarningLine( - `Error calculating the state of the repo. (inner error: ${e}). Continuing without diffing files.` - ); - - return undefined; } - } + }); + + return additionalFilesFromRushProjectConfiguration; } diff --git a/libraries/rush-lib/src/logic/ProjectWatcher.ts b/libraries/rush-lib/src/logic/ProjectWatcher.ts index 824ad800f5b..1d6af513ad9 100644 --- a/libraries/rush-lib/src/logic/ProjectWatcher.ts +++ b/libraries/rush-lib/src/logic/ProjectWatcher.ts @@ -6,20 +6,21 @@ import * as os from 'os'; import * as readline from 'readline'; import { once } from 'events'; import { getRepoRoot } from '@rushstack/package-deps-hash'; -import { Path, type FileSystemStats, FileSystem } from '@rushstack/node-core-library'; +import { AlreadyReportedError, Path, type FileSystemStats, FileSystem } from '@rushstack/node-core-library'; import { Colorize, type ITerminal } from '@rushstack/terminal'; import { Git } from './Git'; -import { ProjectChangeAnalyzer } from './ProjectChangeAnalyzer'; +import type { IInputSnapshot, IInputSnapshotProvider } from './snapshots/InputSnapshot'; import type { RushConfiguration } from '../api/RushConfiguration'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; export interface IProjectWatcherOptions { + snapshotProvider: IInputSnapshotProvider; debounceMs?: number; rushConfiguration: RushConfiguration; projectsToWatch: ReadonlySet; terminal: ITerminal; - initialState?: ProjectChangeAnalyzer | undefined; + initialState?: IInputSnapshot | undefined; } export interface IProjectChangeResult { @@ -30,7 +31,7 @@ export interface IProjectChangeResult { /** * Contains the git hashes for all tracked files in the repo */ - state: ProjectChangeAnalyzer; + state: IInputSnapshot; } export interface IPromptGeneratorFunction { @@ -45,21 +46,22 @@ interface IPathWatchOptions { * This class is for incrementally watching a set of projects in the repository for changes. * * We are manually using fs.watch() instead of `chokidar` because all we want from the file system watcher is a boolean - * signal indicating that "at least 1 file in a watched project changed". We then defer to ProjectChangeAnalyzer (which + * signal indicating that "at least 1 file in a watched project changed". We then defer to IInputSnapshotProvider (which * is responsible for change detection in all incremental builds) to determine what actually chanaged. * * Calling `waitForChange()` will return a promise that resolves when the package-deps of one or * more projects differ from the value the previous time it was invoked. The first time will always resolve with the full selection. */ export class ProjectWatcher { + private readonly _snapshotProvider: IInputSnapshotProvider; private readonly _debounceMs: number; private readonly _repoRoot: string; private readonly _rushConfiguration: RushConfiguration; private readonly _projectsToWatch: ReadonlySet; private readonly _terminal: ITerminal; - private _initialState: ProjectChangeAnalyzer | undefined; - private _previousState: ProjectChangeAnalyzer | undefined; + private _initialState: IInputSnapshot | undefined; + private _previousState: IInputSnapshot | undefined; private _forceChangedProjects: Map = new Map(); private _resolveIfChanged: undefined | (() => Promise); private _getPromptLines: undefined | IPromptGeneratorFunction; @@ -69,7 +71,14 @@ export class ProjectWatcher { public isPaused: boolean = false; public constructor(options: IProjectWatcherOptions) { - const { debounceMs = 1000, rushConfiguration, projectsToWatch, terminal, initialState } = options; + const { + snapshotProvider, + debounceMs = 1000, + rushConfiguration, + projectsToWatch, + terminal, + initialState + } = options; this._debounceMs = debounceMs; this._rushConfiguration = rushConfiguration; @@ -84,6 +93,7 @@ export class ProjectWatcher { this._renderedStatusLines = 0; this._getPromptLines = undefined; + this._snapshotProvider = snapshotProvider; } public pause(): void { @@ -145,7 +155,7 @@ export class ProjectWatcher { return initialChangeResult; } - const previousState: ProjectChangeAnalyzer = initialChangeResult.state; + const previousState: IInputSnapshot = initialChangeResult.state; const repoRoot: string = Path.convertToSlashes(this._rushConfiguration.rushJsonFolder); // Map of path to whether config for the path @@ -170,10 +180,8 @@ export class ProjectWatcher { } } else { for (const project of this._projectsToWatch) { - const projectState: Map = (await previousState._tryGetProjectDependenciesAsync( - project, - this._terminal - ))!; + const projectState: ReadonlyMap = + previousState.getTrackedFileHashesForOperation(project); const prefixLength: number = project.projectFolder.length - repoRoot.length - 1; // Watch files in the root of the project, or @@ -392,9 +400,13 @@ export class ProjectWatcher { * Determines which, if any, projects (within the selection) have new hashes for files that are not in .gitignore */ private async _computeChangedAsync(): Promise { - const state: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this._rushConfiguration); + const state: IInputSnapshot | undefined = await this._snapshotProvider(); - const previousState: ProjectChangeAnalyzer | undefined = this._previousState; + if (!state) { + throw new AlreadyReportedError(); + } + + const previousState: IInputSnapshot | undefined = this._previousState; if (!previousState) { return { @@ -405,10 +417,10 @@ export class ProjectWatcher { const changedProjects: Set = new Set(); for (const project of this._projectsToWatch) { - const [previous, current] = await Promise.all([ - previousState._tryGetProjectDependenciesAsync(project, this._terminal), - state._tryGetProjectDependenciesAsync(project, this._terminal) - ]); + const previous: ReadonlyMap | undefined = + previousState.getTrackedFileHashesForOperation(project); + const current: ReadonlyMap | undefined = + state.getTrackedFileHashesForOperation(project); if (ProjectWatcher._haveProjectDepsChanged(previous, current)) { // May need to detect if the nature of the change will break the process, e.g. changes to package.json @@ -426,7 +438,7 @@ export class ProjectWatcher { }; } - private _commitChanges(state: ProjectChangeAnalyzer): void { + private _commitChanges(state: IInputSnapshot): void { this._previousState = state; if (!this._initialState) { this._initialState = state; @@ -439,8 +451,8 @@ export class ProjectWatcher { * @returns `true` if the maps are different, `false` otherwise */ private static _haveProjectDepsChanged( - prev: Map | undefined, - next: Map | undefined + prev: ReadonlyMap | undefined, + next: ReadonlyMap | undefined ): boolean { if (!prev && !next) { return false; diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index 239634eda37..f4efa538612 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -8,8 +8,6 @@ import { FileSystem, type FolderItem, InternalError, Async } from '@rushstack/no import type { ITerminal } from '@rushstack/terminal'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; -import { RushConstants } from '../RushConstants'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import type { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider'; import type { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider'; @@ -17,14 +15,33 @@ import { TarExecutable } from '../../utilities/TarExecutable'; import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration'; export interface IProjectBuildCacheOptions { + /** + * The repo-wide configuration for the build cache. + */ buildCacheConfiguration: BuildCacheConfiguration; + /** + * The project to be cached. + */ project: RushConfigurationProject; + /** + * Value from rush-project.json + */ projectOutputFolderNames: ReadonlyArray; + /** + * Value from CacheableOperationPlugin + */ additionalProjectOutputFilePaths?: ReadonlyArray; - additionalContext?: Record; - configHash: string; - projectChangeAnalyzer: ProjectChangeAnalyzer; + /** + * The hash of all relevant inputs and configuration that uniquely identifies this execution. + */ + operationStateHash: string; + /** + * The terminal to use for logging. + */ terminal: ITerminal; + /** + * The name of the phase that is being cached. + */ phaseName: string; } @@ -34,11 +51,7 @@ interface IPathsToCache { } export class ProjectBuildCache { - /** - * null === we haven't tried to initialize yet - * undefined === unable to initialize - */ - private static _tarUtilityPromise: Promise | null = null; + private static _tarUtilityPromise: Promise | undefined; private readonly _project: RushConfigurationProject; private readonly _localBuildCacheProvider: FileSystemBuildCacheProvider; @@ -47,7 +60,7 @@ export class ProjectBuildCache { private readonly _cacheWriteEnabled: boolean; private readonly _projectOutputFolderNames: ReadonlyArray; private readonly _additionalProjectOutputFilePaths: ReadonlyArray; - private _cacheId: string | undefined; + private readonly _cacheId: string | undefined; private constructor(cacheId: string | undefined, options: IProjectBuildCacheOptions) { const { @@ -72,7 +85,7 @@ export class ProjectBuildCache { } private static _tryGetTarUtility(terminal: ITerminal): Promise { - if (ProjectBuildCache._tarUtilityPromise === null) { + if (!ProjectBuildCache._tarUtilityPromise) { ProjectBuildCache._tarUtilityPromise = TarExecutable.tryInitializeAsync(terminal); } @@ -83,10 +96,8 @@ export class ProjectBuildCache { return this._cacheId; } - public static async tryGetProjectBuildCacheAsync( - options: IProjectBuildCacheOptions - ): Promise { - const cacheId: string | undefined = await ProjectBuildCache._getCacheIdAsync(options); + public static getProjectBuildCache(options: IProjectBuildCacheOptions): ProjectBuildCache { + const cacheId: string | undefined = ProjectBuildCache._getCacheId(options); return new ProjectBuildCache(cacheId, options); } @@ -150,7 +161,7 @@ export class ProjectBuildCache { const tarUtility: TarExecutable | undefined = await ProjectBuildCache._tryGetTarUtility(terminal); let restoreSuccess: boolean = false; if (tarUtility && localCacheEntryPath) { - const logFilePath: string = this._getTarLogFilePath('untar'); + const logFilePath: string = this._getTarLogFilePath(cacheId, 'untar'); const tarExitCode: number = await tarUtility.tryUntarAsync({ archivePath: localCacheEntryPath, outputFolderPath: projectFolderPath, @@ -207,7 +218,7 @@ export class ProjectBuildCache { const randomSuffix: string = crypto.randomBytes(8).toString('hex'); const tempLocalCacheEntryPath: string = `${finalLocalCacheEntryPath}-${randomSuffix}.temp`; - const logFilePath: string = this._getTarLogFilePath('tar'); + const logFilePath: string = this._getTarLogFilePath(cacheId, 'tar'); const tarExitCode: number = await tarUtility.tryCreateArchiveFromProjectPathsAsync({ archivePath: tempLocalCacheEntryPath, paths: filesToCache.outputFilePaths, @@ -371,84 +382,15 @@ export class ProjectBuildCache { }; } - private _getTarLogFilePath(mode: 'tar' | 'untar'): string { - return path.join(this._project.projectRushTempFolder, `${this._cacheId}.${mode}.log`); + private _getTarLogFilePath(cacheId: string, mode: 'tar' | 'untar'): string { + return path.join(this._project.projectRushTempFolder, `${cacheId}.${mode}.log`); } - private static async _getCacheIdAsync({ - projectChangeAnalyzer, - project, - terminal, - projectOutputFolderNames, - configHash, - additionalContext, - phaseName, - buildCacheConfiguration: { getCacheEntryId } - }: IProjectBuildCacheOptions): Promise { - // The project state hash is calculated in the following method: - // - The current project's hash (see ProjectChangeAnalyzer.getProjectStateHash) is - // calculated and appended to an array - // - The current project's recursive dependency projects' hashes are calculated - // and appended to the array - // - A SHA1 hash is created and the following data is fed into it, in order: - // 1. The JSON-serialized list of output folder names for this - // project (see ProjectBuildCache._projectOutputFolderNames) - // 2. The configHash from the operation's runner - // 3. Each dependency project hash (from the array constructed in previous steps), - // in sorted alphanumerical-sorted order - // - A hex digest of the hash is returned - const projectStates: string[] = []; - const projectsToProcess: Set = new Set(); - projectsToProcess.add(project); - - for (const projectToProcess of projectsToProcess) { - const projectState: string | undefined = await projectChangeAnalyzer._tryGetProjectStateHashAsync( - projectToProcess, - terminal - ); - if (!projectState) { - // If we hit any projects with unknown state, return unknown cache ID - return undefined; - } else { - projectStates.push(projectState); - for (const dependency of projectToProcess.dependencyProjects) { - projectsToProcess.add(dependency); - } - } - } - - const sortedProjectStates: string[] = projectStates.sort(); - const hash: crypto.Hash = crypto.createHash('sha1'); - // This value is used to force cache bust when the build cache algorithm changes - hash.update(`${RushConstants.buildCacheVersion}`); - hash.update(RushConstants.hashDelimiter); - const serializedOutputFolders: string = JSON.stringify(projectOutputFolderNames); - hash.update(serializedOutputFolders); - hash.update(RushConstants.hashDelimiter); - hash.update(configHash); - hash.update(RushConstants.hashDelimiter); - if (additionalContext) { - for (const key of Object.keys(additionalContext).sort()) { - // Add additional context keys and values. - // - // This choice (to modify the hash for every key regardless of whether a value is set) implies - // that just _adding_ an env var to the list of dependsOnEnvVars will modify its hash. This - // seems appropriate, because this behavior is consistent whether or not the env var happens - // to have a value. - hash.update(`${key}=${additionalContext[key]}`); - hash.update(RushConstants.hashDelimiter); - } - } - for (const projectHash of sortedProjectStates) { - hash.update(projectHash); - hash.update(RushConstants.hashDelimiter); - } - - const projectStateHash: string = hash.digest('hex'); - - return getCacheEntryId({ - projectName: project.packageName, - projectStateHash, + private static _getCacheId(options: IProjectBuildCacheOptions): string | undefined { + const { buildCacheConfiguration, project: { packageName }, operationStateHash, phaseName } = options; + return buildCacheConfiguration.getCacheEntryId({ + projectName: packageName, + projectStateHash: operationStateHash, phaseName }); } diff --git a/libraries/rush-lib/src/logic/buildCache/getHashesForGlobsAsync.ts b/libraries/rush-lib/src/logic/buildCache/getHashesForGlobsAsync.ts deleted file mode 100644 index 628bc03d800..00000000000 --- a/libraries/rush-lib/src/logic/buildCache/getHashesForGlobsAsync.ts +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { getGitHashForFiles } from '@rushstack/package-deps-hash'; -import * as path from 'path'; -import type { IRawRepoState } from '../ProjectChangeAnalyzer'; - -async function expandGlobPatternsAsync( - globPatterns: Iterable, - packagePath: string -): Promise { - const { default: glob } = await import('fast-glob'); - const matches: string[] = await glob(Array.from(globPatterns), { - cwd: packagePath, - onlyFiles: true, - // We want to keep path's type unchanged, - // i.e. if the pattern was a relative path, then matched paths should also be relative paths - // if the pattern was an absolute path, then matched paths should also be absolute paths - // - // We are doing this because these paths are going to be used to calculate a hash for the build cache and some users - // might choose to depend on global files (e.g. `/etc/os-release`) and some might choose to depend on local files - // (e.g. `../path/to/workspace/file`) - // - // In both cases we want that path to the resource would be the same on all machines, - // regardless of what is the current working directory. - // - // That being said, we want to keep `absolute` option here as false: - absolute: false - }); - - if (matches.length === 0) { - throw new Error( - `Couldn't find any files matching provided glob patterns: ["${Array.from(globPatterns).join('", "')}"].` - ); - } - - return matches; -} - -interface IKnownHashesResult { - foundPaths: Map; - missingPaths: string[]; -} - -function getKnownHashes( - filePaths: string[], - packagePath: string, - repoState: IRawRepoState -): IKnownHashesResult { - const missingPaths: string[] = []; - const foundPaths: Map = new Map(); - - for (const filePath of filePaths) { - const absolutePath: string = path.isAbsolute(filePath) ? filePath : path.join(packagePath, filePath); - - /** - * We are using RegExp here to prevent false positives in the following string.replace function - * - `^` anchor makes sure that we are replacing only the beginning of the string - * - extra `/` makes sure that we are remove extra slash from the relative path - */ - const gitFilePath: string = absolutePath.replace(new RegExp('^' + repoState.rootDir + '/'), ''); - const foundHash: string | undefined = repoState.rawHashes.get(gitFilePath); - - if (foundHash) { - foundPaths.set(filePath, foundHash); - } else { - missingPaths.push(filePath); - } - } - - return { foundPaths, missingPaths }; -} - -export async function getHashesForGlobsAsync( - globPatterns: Iterable, - packagePath: string, - repoState: IRawRepoState | undefined -): Promise> { - const filePaths: string[] = await expandGlobPatternsAsync(globPatterns, packagePath); - - if (!repoState) { - return getGitHashForFiles(filePaths, packagePath); - } - - const { foundPaths, missingPaths } = getKnownHashes(filePaths, packagePath, repoState); - const calculatedHashes: Map = getGitHashForFiles(missingPaths, packagePath); - - /** - * We want to keep the order of the output the same regardless whether the file was already - * hashed by git or not (as this can change, e.g. due to .gitignore). - * Therefore we will populate our final hashes map in the same order as `filePaths`. - */ - const result: Map = new Map(); - for (const filePath of filePaths) { - const hash: string | undefined = foundPaths.get(filePath) || calculatedHashes.get(filePath); - if (!hash) { - // Sanity check -- this should never happen - throw new Error(`Failed to calculate hash of file: "${filePath}"`); - } - result.set(filePath, hash); - } - - return result; -} diff --git a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts index c92d2965340..1ba67061f54 100644 --- a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts +++ b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts @@ -5,7 +5,6 @@ import { StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; import type { BuildCacheConfiguration } from '../../../api/BuildCacheConfiguration'; import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; -import { ProjectChangeAnalyzer } from '../../ProjectChangeAnalyzer'; import type { IGenerateCacheEntryIdOptions } from '../CacheEntryId'; import type { FileSystemBuildCacheProvider } from '../FileSystemBuildCacheProvider'; @@ -18,15 +17,10 @@ interface ITestOptions { } describe(ProjectBuildCache.name, () => { - async function prepareSubject(options: Partial): Promise { + function prepareSubject(options: Partial): ProjectBuildCache { const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - const projectChangeAnalyzer = { - [ProjectChangeAnalyzer.prototype._tryGetProjectStateHashAsync.name]: async () => { - return 'state_hash'; - } - } as unknown as ProjectChangeAnalyzer; - const subject: ProjectBuildCache | undefined = await ProjectBuildCache.tryGetProjectBuildCacheAsync({ + const subject: ProjectBuildCache = ProjectBuildCache.getProjectBuildCache({ buildCacheConfiguration: { buildCacheEnabled: options.hasOwnProperty('enabled') ? options.enabled : true, getCacheEntryId: (opts: IGenerateCacheEntryIdOptions) => @@ -42,8 +36,7 @@ describe(ProjectBuildCache.name, () => { projectRelativeFolder: 'apps/acme-wizard', dependencyProjects: [] } as unknown as RushConfigurationProject, - configHash: 'build', - projectChangeAnalyzer, + operationStateHash: 'build', terminal, phaseName: 'build' }); @@ -51,12 +44,10 @@ describe(ProjectBuildCache.name, () => { return subject; } - describe(ProjectBuildCache.tryGetProjectBuildCacheAsync.name, () => { - it('returns a ProjectBuildCache with a calculated cacheId value', async () => { - const subject: ProjectBuildCache = (await prepareSubject({}))!; - expect(subject['_cacheId']).toMatchInlineSnapshot( - `"acme-wizard/1926f30e8ed24cb47be89aea39e7efd70fcda075"` - ); + describe(ProjectBuildCache.getProjectBuildCache.name, () => { + it('returns a ProjectBuildCache with a calculated cacheId value', () => { + const subject: ProjectBuildCache = prepareSubject({}); + expect(subject['_cacheId']).toMatchInlineSnapshot(`"acme-wizard/build"`); }); }); }); diff --git a/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts b/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts index ab835024798..bc3f223c0b4 100644 --- a/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts @@ -40,13 +40,13 @@ export class BuildPlanPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { const terminal: ITerminal = this._terminal; - hooks.beforeExecuteOperations.tapPromise(PLUGIN_NAME, createBuildPlan); + hooks.beforeExecuteOperations.tap(PLUGIN_NAME, createBuildPlan); - async function createBuildPlan( + function createBuildPlan( recordByOperation: Map, context: IExecuteOperationsContext - ): Promise { - const { projectConfigurations, projectChangeAnalyzer } = context; + ): void { + const { projectConfigurations, inputSnapshot } = context; const disjointSet: DisjointSet = new DisjointSet(); const operations: Operation[] = [...recordByOperation.keys()]; for (const operation of operations) { @@ -56,13 +56,14 @@ export class BuildPlanPlugin implements IPhasedCommandPlugin { Operation, IBuildPlanOperationCacheContext >(); + for (const operation of operations) { const { associatedProject, associatedPhase } = operation; if (associatedProject && associatedPhase) { const projectConfiguration: RushProjectConfiguration | undefined = projectConfigurations.get(associatedProject); - const fileHashes: Map | undefined = - await projectChangeAnalyzer._tryGetProjectDependenciesAsync(associatedProject, terminal); + const fileHashes: ReadonlyMap | undefined = + inputSnapshot?.getTrackedFileHashesForOperation(associatedProject, associatedPhase.name); if (!fileHashes) { continue; } diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index dd5812e5aa6..6efbd7a1353 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -2,7 +2,7 @@ // See LICENSE in the project root for license information. import * as crypto from 'crypto'; -import { Async, InternalError, NewlineKind, Sort } from '@rushstack/node-core-library'; +import { InternalError, NewlineKind, Sort } from '@rushstack/node-core-library'; import { CollatedTerminal, type CollatedWriter } from '@rushstack/stream-collator'; import { DiscardStdoutTransform, TextRewriterTransform } from '@rushstack/terminal'; import { SplitterTransform, type TerminalWritable, type ITerminal, Terminal } from '@rushstack/terminal'; @@ -12,8 +12,7 @@ import { OperationStatus } from './OperationStatus'; import { CobuildLock, type ICobuildCompletedState } from '../cobuild/CobuildLock'; import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; import { RushConstants } from '../RushConstants'; -import { type IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; -import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; +import type { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { initializeProjectLogFilesAsync, getProjectLogFilePaths, @@ -33,11 +32,11 @@ import type { PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; import type { IPhase } from '../../api/CommandLineConfiguration'; -import type { IRawRepoState, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import type { OperationMetadataManager } from './OperationMetadataManager'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; import type { OperationExecutionRecord } from './OperationExecutionRecord'; +import type { IInputSnapshot } from '../snapshots/InputSnapshot'; const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; const PERIODIC_CALLBACK_INTERVAL_IN_SECONDS: number = 10; @@ -51,10 +50,11 @@ export interface IOperationBuildCacheContext { isCacheWriteAllowed: boolean; isCacheReadAllowed: boolean; - projectChangeAnalyzer: ProjectChangeAnalyzer; - projectBuildCache: ProjectBuildCache | undefined; + stateHash: string; + + operationBuildCache: ProjectBuildCache | undefined; cacheDisabledReason: string | undefined; - operationSettings: IOperationSettings | undefined; + outputFolderNames: ReadonlyArray | undefined; cobuildLock: CobuildLock | undefined; @@ -87,101 +87,131 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } public apply(hooks: PhasedCommandHooks): void { - const { allowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration, terminal } = - this._options; + const { allowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration } = this._options; - hooks.beforeExecuteOperations.tapPromise( + hooks.beforeExecuteOperations.tap( PLUGIN_NAME, - async ( + ( recordByOperation: Map, context: IExecuteOperationsContext - ): Promise => { - const { isIncrementalBuildAllowed, projectChangeAnalyzer, projectConfigurations, isInitial } = - context; + ): void => { + const { isIncrementalBuildAllowed, inputSnapshot, projectConfigurations, isInitial } = context; + + if (!inputSnapshot) { + throw new Error( + `Build cache is only supported if running in a Git repository. Either disable the build cache or run Rush in a Git repository.` + ); + } + + // This redefinition is necessary due to limitations in TypeScript's control flow analysis, due to the nested closure. + const definitelyDefinedInputSnapshot: IInputSnapshot = inputSnapshot; const disjointSet: DisjointSet | undefined = cobuildConfiguration?.cobuildFeatureEnabled ? new DisjointSet() : undefined; - for (const operation of recordByOperation.keys()) { - if ( - operation.settings?.allowCobuildWithoutCache && - !cobuildConfiguration?.cobuildWithoutCacheAllowed - ) { - throw new Error( - `Operation ${operation.name} is not allowed to run without the cobuild orchestration experiment enabled. You must enable the "allowCobuildWithoutCache" experiment in experiments.json.` - ); - } - if ( - operation.settings?.allowCobuildWithoutCache && - !operation.settings.disableBuildCacheForOperation - ) { - throw new Error( - `Operation ${operation.name} must have disableBuildCacheForOperation set to true when using the cobuild orchestration experiment. This is to prevent implicit cache dependencies for this operation.` - ); + const hashByOperation: Map = new Map(); + // Build cache hashes are computed up front to ensure stability and to catch configuration errors early. + function getOrCreateOperationHash(operation: Operation): string { + const cachedHash: string | undefined = hashByOperation.get(operation); + if (cachedHash !== undefined) { + return cachedHash; } - } - await Async.forEachAsync( - recordByOperation.keys(), - async (operation: Operation) => { - const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation; - if (!associatedProject || !associatedPhase || !runner) { - return; - } + // Examples of data in the config hash: + // - CLI parameters (ShellOperationRunner) + const configHash: string | undefined = operation.runner?.getConfigHash(); + + const { associatedProject, associatedPhase } = operation; + // Examples of data in the local state hash: + // - Environment variables specified in `dependsOnEnvVars` + // - Git hashes of tracked files in the associated project + // - Git hash of the shrinkwrap file for the project + // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) + const localStateHash: string | undefined = + associatedProject && + definitelyDefinedInputSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name); + + // The final state hashes of operation dependencies are factored into the hash to ensure that any + // state changes in dependencies will invalidate the cache. + const dependencyHashes: string[] = Array.from(operation.dependencies, getDependencyHash).sort(); + + const hasher: crypto.Hash = crypto.createHash('sha1'); + // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content + // of the build cache. + hasher.update(`${RushConstants.buildCacheVersion}`); + + for (const dependencyHash of dependencyHashes) { + hasher.update(dependencyHash); + } - const { name: phaseName } = associatedPhase; + if (localStateHash) { + hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`); + } - const projectConfiguration: RushProjectConfiguration | undefined = - projectConfigurations.get(associatedProject); + if (configHash) { + hasher.update(`${RushConstants.hashDelimiter}${configHash}`); + } - // This value can *currently* be cached per-project, but in the future the list of files will vary - // depending on the selected phase. - const fileHashes: Map | undefined = - await projectChangeAnalyzer._tryGetProjectDependenciesAsync(associatedProject, terminal); + const hashString: string = hasher.digest('hex'); - if (!fileHashes) { - throw new Error( - `Build cache is only supported if running in a Git repository. Either disable the build cache or run Rush in a Git repository.` - ); - } + hashByOperation.set(operation, hashString); + return hashString; + } - const cacheDisabledReason: string | undefined = - RushProjectConfiguration.getCacheDisabledReasonForProject({ - projectConfiguration, - phaseName: phaseName, - isNoOp: operation.isNoOp, - trackedFileNames: fileHashes.keys() - }); + function getDependencyHash(operation: Operation): string { + return `${RushConstants.hashDelimiter}${operation.name}=${getOrCreateOperationHash(operation)}`; + } - disjointSet?.add(operation); - - const buildCacheContext: IOperationBuildCacheContext = { - // Supports cache writes by default for initial operations. - // Don't write during watch runs for performance reasons (and to avoid flooding the cache) - isCacheWriteAllowed: isInitial, - isCacheReadAllowed: isIncrementalBuildAllowed, - projectBuildCache: undefined, - projectChangeAnalyzer, - operationSettings, - cacheDisabledReason, - cobuildLock: undefined, - cobuildClusterId: undefined, - buildCacheTerminal: undefined, - buildCacheTerminalWritable: undefined, - periodicCallback: new PeriodicCallback({ - interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000 - }), - cacheRestored: false, - isCacheReadAttempted: false - }; - // Upstream runners may mutate the property of build cache context for downstream runners - this._buildCacheContextByOperation.set(operation, buildCacheContext); - }, - { - concurrency: 10 + for (const operation of recordByOperation.keys()) { + const { associatedProject, associatedPhase, runner } = operation; + if (!associatedProject || !associatedPhase || !runner) { + return; } - ); + + const { name: phaseName } = associatedPhase; + + const projectConfiguration: RushProjectConfiguration | undefined = + projectConfigurations.get(associatedProject); + + // This value can *currently* be cached per-project, but in the future the list of files will vary + // depending on the selected phase. + const fileHashes: ReadonlyMap | undefined = + inputSnapshot.getTrackedFileHashesForOperation(associatedProject, phaseName); + const stateHash: string = getOrCreateOperationHash(operation); + + const cacheDisabledReason: string | undefined = projectConfiguration + ? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName, operation.isNoOp) + : `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + + 'or one provided by a rig, so it does not support caching.'; + + const outputFolderNames: ReadonlyArray | undefined = + projectConfiguration?.operationSettingsByOperationName.get(phaseName)?.outputFolderNames; + + disjointSet?.add(operation); + + const buildCacheContext: IOperationBuildCacheContext = { + // Supports cache writes by default for initial operations. + // Don't write during watch runs for performance reasons (and to avoid flooding the cache) + isCacheWriteAllowed: isInitial, + isCacheReadAllowed: isIncrementalBuildAllowed, + operationBuildCache: undefined, + outputFolderNames, + stateHash, + cacheDisabledReason, + cobuildLock: undefined, + cobuildClusterId: undefined, + buildCacheTerminal: undefined, + buildCacheTerminalWritable: undefined, + periodicCallback: new PeriodicCallback({ + interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000 + }), + cacheRestored: false, + isCacheReadAttempted: false + }; + // Upstream runners may mutate the property of build cache context for downstream runners + this._buildCacheContextByOperation.set(operation, buildCacheContext); + } if (disjointSet) { clusterOperations(disjointSet, this._buildCacheContextByOperation); @@ -272,14 +302,12 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } const buildCacheTerminal: ITerminal = buildCacheContext.buildCacheTerminal; - const configHash: string = runner.getConfigHash() || ''; - let projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ + let projectBuildCache: ProjectBuildCache | undefined = this._tryGetProjectBuildCache({ buildCacheContext, buildCacheConfiguration, rushProject: project, phase, - configHash, terminal: buildCacheTerminal, operationMetadataManager, operation: record.operation @@ -301,7 +329,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheContext, rushProject: project, phase, - configHash, terminal: buildCacheTerminal, operationMetadataManager }); @@ -456,7 +483,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return; } - const { cobuildLock, projectBuildCache, isCacheWriteAllowed, buildCacheTerminal, cacheRestored } = + const { cobuildLock, operationBuildCache, isCacheWriteAllowed, buildCacheTerminal, cacheRestored } = buildCacheContext; try { @@ -523,8 +550,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // If the command is successful, we can calculate project hash, and no dependencies were skipped, // write a new cache entry. - if (!setCacheEntryPromise && taskIsSuccessful && isCacheWriteAllowed && projectBuildCache) { - setCacheEntryPromise = () => projectBuildCache.trySetCacheEntryAsync(buildCacheTerminal); + if (!setCacheEntryPromise && taskIsSuccessful && isCacheWriteAllowed && operationBuildCache) { + setCacheEntryPromise = () => operationBuildCache.trySetCacheEntryAsync(buildCacheTerminal); } if (!cacheRestored) { const cacheWriteSuccess: boolean | undefined = await setCacheEntryPromise?.(); @@ -592,12 +619,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext; } - private async _tryGetProjectBuildCacheAsync({ + private _tryGetProjectBuildCache({ buildCacheConfiguration, buildCacheContext, rushProject, phase, - configHash, terminal, operationMetadataManager, operation @@ -606,52 +632,39 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheConfiguration: BuildCacheConfiguration | undefined; rushProject: RushConfigurationProject; phase: IPhase; - configHash: string; terminal: ITerminal; operationMetadataManager: OperationMetadataManager | undefined; operation: Operation; - }): Promise { - if (!buildCacheContext.projectBuildCache) { + }): ProjectBuildCache | undefined { + if (!buildCacheContext.operationBuildCache) { const { cacheDisabledReason } = buildCacheContext; if (cacheDisabledReason && !operation.settings?.allowCobuildWithoutCache) { terminal.writeVerboseLine(cacheDisabledReason); return; } - const { operationSettings, projectChangeAnalyzer } = buildCacheContext; - if (!operationSettings || !buildCacheConfiguration) { + const { outputFolderNames, stateHash: operationStateHash } = buildCacheContext; + if (!outputFolderNames || !buildCacheConfiguration) { // Unreachable, since this will have set `cacheDisabledReason`. return; } - const projectOutputFolderNames: ReadonlyArray = operationSettings.outputFolderNames || []; const additionalProjectOutputFilePaths: ReadonlyArray = operationMetadataManager?.relativeFilepaths || []; - const additionalContext: Record = {}; - - await updateAdditionalContextAsync({ - operationSettings, - additionalContext, - projectChangeAnalyzer, - terminal, - rushProject - }); // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent - buildCacheContext.projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCacheAsync({ + buildCacheContext.operationBuildCache = ProjectBuildCache.getProjectBuildCache({ project: rushProject, - projectOutputFolderNames, + projectOutputFolderNames: outputFolderNames, additionalProjectOutputFilePaths, - additionalContext, buildCacheConfiguration, terminal, - configHash, - projectChangeAnalyzer, + operationStateHash, phaseName: phase.name }); } - return buildCacheContext.projectBuildCache; + return buildCacheContext.operationBuildCache; } // Get a ProjectBuildCache only cache/restore log files @@ -659,7 +672,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheContext, rushProject, terminal, - configHash, buildCacheConfiguration, cobuildConfiguration, phase, @@ -670,7 +682,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { cobuildConfiguration: CobuildConfiguration; rushProject: RushConfigurationProject; phase: IPhase; - configHash: string; terminal: ITerminal; operationMetadataManager: OperationMetadataManager | undefined; }): Promise { @@ -678,44 +689,34 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return; } - const { operationSettings, projectChangeAnalyzer } = buildCacheContext; + const { outputFolderNames, stateHash } = buildCacheContext; - const projectOutputFolderNames: ReadonlyArray = operationSettings?.outputFolderNames ?? []; const additionalProjectOutputFilePaths: ReadonlyArray = operationMetadataManager?.relativeFilepaths || []; - const additionalContext: Record = { - // Force the cache to be a log files only cache - logFilesOnly: '1' - }; + + const hasher: crypto.Hash = crypto.createHash('sha1'); + hasher.update(stateHash); + if (cobuildConfiguration.cobuildContextId) { - additionalContext.cobuildContextId = cobuildConfiguration.cobuildContextId; + hasher.update(`\ncobuildContextId=${cobuildConfiguration.cobuildContextId}`); } - if (operationSettings) { - await updateAdditionalContextAsync({ - operationSettings, - additionalContext, - projectChangeAnalyzer, - terminal, - rushProject - }); - } + hasher.update(`\nlogFilesOnly=1`); - const projectBuildCache: ProjectBuildCache | undefined = - await ProjectBuildCache.tryGetProjectBuildCacheAsync({ - project: rushProject, - projectOutputFolderNames, - additionalProjectOutputFilePaths, - additionalContext, - buildCacheConfiguration, - terminal, - configHash, - projectChangeAnalyzer, - phaseName: phase.name - }); + const operationStateHash: string = hasher.digest('hex'); + + const projectBuildCache: ProjectBuildCache = ProjectBuildCache.getProjectBuildCache({ + project: rushProject, + projectOutputFolderNames: outputFolderNames || [], + additionalProjectOutputFilePaths, + buildCacheConfiguration, + terminal, + operationStateHash, + phaseName: phase.name + }); // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent - buildCacheContext.projectBuildCache = projectBuildCache; + buildCacheContext.operationBuildCache = projectBuildCache; return projectBuildCache; } @@ -847,46 +848,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext.buildCacheTerminalWritable; } } -async function updateAdditionalContextAsync({ - operationSettings, - additionalContext, - projectChangeAnalyzer, - terminal, - rushProject -}: { - operationSettings: IOperationSettings; - additionalContext: Record; - projectChangeAnalyzer: ProjectChangeAnalyzer; - terminal: ITerminal; - rushProject: RushConfigurationProject; -}): Promise { - if (operationSettings.dependsOnEnvVars) { - for (const varName of operationSettings.dependsOnEnvVars) { - additionalContext['$' + varName] = process.env[varName] || ''; - } - } - - if (operationSettings.dependsOnAdditionalFiles) { - const repoState: IRawRepoState | undefined = - await projectChangeAnalyzer._ensureInitializedAsync(terminal); - - const additionalFiles: Map = await getHashesForGlobsAsync( - operationSettings.dependsOnAdditionalFiles, - rushProject.projectFolder, - repoState - ); - - terminal.writeDebugLine( - `Including additional files to calculate build cache hash:\n ${Array.from(additionalFiles.keys()).join( - '\n ' - )} ` - ); - - for (const [filePath, fileHash] of additionalFiles) { - additionalContext['file://' + filePath] = fileHash; - } - } -} export function clusterOperations( initialClusters: DisjointSet, diff --git a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts index 9b8045f1922..b6b0fcf68f8 100644 --- a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts @@ -3,7 +3,7 @@ import path from 'node:path'; -import { Async, FileSystem, JsonFile, type JsonObject } from '@rushstack/node-core-library'; +import { FileSystem, JsonFile, type JsonObject } from '@rushstack/node-core-library'; import { PrintUtilities, Colorize, type ITerminal } from '@rushstack/terminal'; import type { Operation } from './Operation'; @@ -66,15 +66,16 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { const { terminal, changedProjectsOnly, isIncrementalBuildAllowed, allowWarningsInSuccessfulBuild } = this._options; - hooks.beforeExecuteOperations.tapPromise( + hooks.beforeExecuteOperations.tap( PLUGIN_NAME, - async ( + ( operations: ReadonlyMap, - { projectChangeAnalyzer }: IExecuteOperationsContext - ): Promise => { + context: IExecuteOperationsContext + ): void => { let logGitWarning: boolean = false; + const { inputSnapshot } = context; - await Async.forEachAsync(operations.values(), async (record: IOperationExecutionResult) => { + for (const record of operations.values()) { const { operation } = record; const { associatedProject, runner, logFilenameIdentifier } = operation; if (!associatedProject || !runner) { @@ -100,8 +101,11 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { let packageDeps: IProjectDeps | undefined; try { - const fileHashes: Map | undefined = - await projectChangeAnalyzer._tryGetProjectDependenciesAsync(associatedProject, terminal); + const fileHashes: ReadonlyMap | undefined = + inputSnapshot?.getTrackedFileHashesForOperation( + associatedProject, + operation.associatedPhase?.name + ); if (!fileHashes) { logGitWarning = true; @@ -134,7 +138,7 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { packageDeps, allowSkip: isIncrementalBuildAllowed }); - }); + } if (logGitWarning) { // To test this code path: diff --git a/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts index 9401819a483..3a7019c4070 100644 --- a/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts @@ -22,9 +22,9 @@ import { PhasedOperationPlugin } from '../PhasedOperationPlugin'; import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; import { RushConstants } from '../../RushConstants'; import { MockOperationRunner } from './MockOperationRunner'; -import { ProjectChangeAnalyzer } from '../../ProjectChangeAnalyzer'; import path from 'path'; import type { ICommandLineJson } from '../../../api/CommandLineJson'; +import type { IInputSnapshot } from '../../snapshots/InputSnapshot'; describe(BuildPlanPlugin.name, () => { const rushJsonFile: string = path.resolve(__dirname, `../../test/workspaceRepo/rush.json`); @@ -104,12 +104,13 @@ describe(BuildPlanPlugin.name, () => { const hooks: PhasedCommandHooks = new PhasedCommandHooks(); new BuildPlanPlugin(terminal).apply(hooks); - const context: Pick = { - projectChangeAnalyzer: { - [ProjectChangeAnalyzer.prototype._tryGetProjectDependenciesAsync.name]: async () => { - return new Map(); - } - } as unknown as ProjectChangeAnalyzer, + const inputSnapshot: Pick = { + getTrackedFileHashesForOperation() { + return new Map(); + } + }; + const context: Pick = { + inputSnapshot: inputSnapshot as unknown as IInputSnapshot, projectConfigurations: new Map() }; const buildCommand: IPhasedCommandConfig = commandLineConfiguration.commands.get( diff --git a/libraries/rush-lib/src/logic/snapshots/InputSnapshot.ts b/libraries/rush-lib/src/logic/snapshots/InputSnapshot.ts new file mode 100644 index 00000000000..bfc0463cb8c --- /dev/null +++ b/libraries/rush-lib/src/logic/snapshots/InputSnapshot.ts @@ -0,0 +1,455 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as path from 'node:path'; +import { createHash, type Hash } from 'node:crypto'; +import ignore, { type Ignore } from 'ignore'; + +import { type IReadonlyLookupByPath, LookupByPath } from '@rushstack/lookup-by-path'; +import { InternalError, Path, Sort } from '@rushstack/node-core-library'; + +import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import type { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; +import { RushConstants } from '../RushConstants'; + +/** + * @beta + */ +export type IRushConfigurationProjectForSnapshot = Pick< + RushConfigurationProject, + 'projectFolder' | 'projectRelativeFolder' +>; + +/** + * @internal + */ +export interface IRushSnapshotProjectMetadata { + /** + * The contents of rush-project.json for the project, if available + */ + projectConfig?: RushProjectConfiguration; + /** + * A map of operation name to additional files that should be included in the hash for that operation. + */ + additionalFilesByOperationName?: ReadonlyMap>; +} + +interface IInternalRushSnapshotProjectMetadata extends IRushSnapshotProjectMetadata { + /** + * Cached filter of files that are not ignored by the project's `incrementalBuildIgnoredGlobs`. + * @param filePath - The path to the file to check + * @returns true if the file path is an input to all operations in the project, false otherwise + */ + projectFilePathFilter?: (filePath: string) => boolean; + /** + * The cached Git hashes for all files in the project folder. + */ + hashes: Map; + /** + * Cached hashes for all files in the project folder, including additional files. + * Upon calculating this map, input-output file collisions are detected. + */ + fileHashesByOperationName: Map>; + /** + * The flattened state hash for each operation name, where the key "undefined" represents no particular operation. + */ + hashByOperationName: Map; + /** + * The project relative folder, which is a prefix in all relative paths. + */ + relativePrefix: string; +} + +export type IRushSnapshotProjectMetadataMap = ReadonlyMap< + IRushConfigurationProjectForSnapshot, + IRushSnapshotProjectMetadata +>; + +/** + * A function that can be invoked to get the current Rush snapshot. + * Binds the project configurations when created. + * + * @beta + */ +export interface IInputSnapshotProvider { + /** + * Compute a new snapshot of the current state of the repository as of the current moment. + * Captures the state of the environment, tracked files, and additional files. + */ + (): Promise; +} + +/** + * The parameters for constructing an {@link InputSnapshot}. + * @internal + */ +export interface IRushSnapshotParameters { + /** + * Hashes for files selected by `dependsOnAdditionalFiles`. + * Separated out to prevent being auto-assigned to a project. + */ + additionalHashes?: ReadonlyMap; + /** + * The environment to use for `dependsOnEnvVars`. By default performs a snapshot of process.env upon construction. + * @defaultValue \{ ...process.env \} + */ + environment?: Record; + /** + * File paths (keys into additionalHashes or hashes) to be included as part of every operation's dependencies. + */ + globalAdditionalFiles?: Iterable; + /** + * The hashes of all tracked files in the repository. + */ + hashes: ReadonlyMap; + /** + * Optimized lookup engine used to route `hashes` to individual projects. + */ + lookupByPath: IReadonlyLookupByPath; + /** + * Metadata for each project. + */ + projectMap: IRushSnapshotProjectMetadataMap; + /** + * The directory that all relative paths are relative to. + */ + rootDir: string; +} + +const { hashDelimiter } = RushConstants; + +/** + * Represents a synchronously-queryable in-memory snapshot of the state of the inputs to a Rush repository. + * + * The methods on this interface are idempotent and will return the same result regardless of when they are executed. + * @beta + */ +export interface IInputSnapshot { + /** + * The raw hashes of all tracked files in the repository. + */ + readonly hashes: ReadonlyMap; + + /** + * The directory that all paths in `hashes` are relative to. + */ + readonly rootDirectory: string; + + /** + * Gets the map of file paths to Git hashes that will be used to compute the local state hash of the operation. + * Exposed separately from the final state hash to facilitate detailed change detection. + * + * @param project - The Rush project to get hashes for + * @param operationName - The name of the operation (phase) to get hashes for. If omitted, returns a default set for the project, as used for bulk commands. + * @returns A map of file name to Git hash. For local files paths will be relative. Configured additional files may be absolute paths. + */ + getTrackedFileHashesForOperation( + project: IRushConfigurationProjectForSnapshot, + operationName?: string + ): ReadonlyMap; + + /** + * Gets the state hash for the files owned by this operation, including the resolutions of package.json dependencies. This will later be combined with the hash of + * the command being executed and the final hashes of the operation's dependencies to compute the final hash for the operation. + * @param project - The Rush project to compute the state hash for + * @param operationName - The name of the operation (phase) to get hashes for. If omitted, returns a generic hash for the whole project, as used for bulk commands. + * @returns The local state hash for the project. This is a hash of the environment, the project's tracked files, and any additional files. + */ + getOperationOwnStateHash(project: IRushConfigurationProjectForSnapshot, operationName?: string): string; +} + +/** + * Represents a synchronously-queryable in-memory snapshot of the state of the inputs to a Rush repository. + * Any asynchronous work needs to be performed by the caller and the results passed to the constructor. + * + * @remarks + * All operations on this class will return the same result regardless of when they are executed. + * + * @internal + */ +export class InputSnapshot implements IInputSnapshot { + /** + * {@inheritdoc IInputSnapshot.hashes} + */ + public readonly hashes: ReadonlyMap; + /** + * {@inheritdoc IInputSnapshot.rootDirectory} + */ + public readonly rootDirectory: string; + + /** + * The metadata for each project. This is a superset of the information in `projectMap` and includes caching of queries. + */ + private readonly _projectMetadataMap: Map< + IRushConfigurationProjectForSnapshot, + IInternalRushSnapshotProjectMetadata + >; + /** + * Hashes of files to be included in all result sets. + */ + private readonly _globalAdditionalHashes: ReadonlyMap | undefined; + /** + * Hashes for files selected by `dependsOnAdditionalFiles`. + */ + private readonly _additionalHashes: ReadonlyMap | undefined; + /** + * The environment to use for `dependsOnEnvVars`. + */ + private readonly _environment: Record; + + /** + * + * @param params - The parameters for the snapshot + * @internal + */ + public constructor(params: IRushSnapshotParameters) { + const { + additionalHashes, + environment = { ...process.env }, + globalAdditionalFiles, + hashes, + lookupByPath, + rootDir + } = params; + const projectMetadataMap: Map< + IRushConfigurationProjectForSnapshot, + IInternalRushSnapshotProjectMetadata + > = new Map(); + const createInternalRecord = ( + project: IRushConfigurationProjectForSnapshot, + baseRecord: IRushSnapshotProjectMetadata | undefined + ): IInternalRushSnapshotProjectMetadata => { + return { + // Data from the caller + projectConfig: baseRecord?.projectConfig, + additionalFilesByOperationName: baseRecord?.additionalFilesByOperationName, + + // Caches + hashes: new Map(), + hashByOperationName: new Map(), + fileHashesByOperationName: new Map(), + relativePrefix: getRelativePrefix(project, rootDir) + }; + }; + for (const [project, record] of params.projectMap) { + projectMetadataMap.set(project, createInternalRecord(project, record)); + } + + // Route hashes to individual projects + for (const [file, hash] of hashes) { + const project: IRushConfigurationProjectForSnapshot | undefined = lookupByPath.findChildPath(file); + if (!project) { + continue; + } + + let record: IInternalRushSnapshotProjectMetadata | undefined = projectMetadataMap.get(project); + if (!record) { + projectMetadataMap.set(project, (record = createInternalRecord(project, undefined))); + } + + record.hashes.set(file, hash); + } + + let globalAdditionalHashes: Map | undefined; + if (globalAdditionalFiles) { + globalAdditionalHashes = new Map(); + const sortedAdditionalFiles: string[] = Array.from(globalAdditionalFiles).sort(); + for (const file of sortedAdditionalFiles) { + const hash: string | undefined = hashes.get(file); + if (!hash) { + throw new Error(`Hash not found for global file: "${file}"`); + } + const owningProject: IRushConfigurationProjectForSnapshot | undefined = + lookupByPath.findChildPath(file); + if (owningProject) { + throw new InternalError( + `Requested global additional file "${file}" is owned by project in "${owningProject.projectRelativeFolder}". Declare a project dependency instead.` + ); + } + globalAdditionalHashes.set(file, hash); + } + } + + for (const record of projectMetadataMap.values()) { + // Ensure stable ordering. + Sort.sortMapKeys(record.hashes); + } + + this._projectMetadataMap = projectMetadataMap; + this._additionalHashes = additionalHashes; + this._globalAdditionalHashes = globalAdditionalHashes; + // Snapshot the environment so that queries are not impacted by when they happen + this._environment = environment; + this.hashes = hashes; + this.rootDirectory = rootDir; + } + + /** + * {@inheritdoc} + */ + public getTrackedFileHashesForOperation( + project: IRushConfigurationProjectForSnapshot, + operationName?: string + ): ReadonlyMap { + const record: IInternalRushSnapshotProjectMetadata | undefined = this._projectMetadataMap.get(project); + if (!record) { + throw new InternalError(`No information available for project at ${project.projectFolder}`); + } + + const { fileHashesByOperationName } = record; + let hashes: Map | undefined = fileHashesByOperationName.get(operationName); + if (!hashes) { + hashes = new Map(); + fileHashesByOperationName.set(operationName, hashes); + // TODO: Support incrementalBuildIgnoredGlobs per-operation + const filter: (filePath: string) => boolean = this._getOrCreateProjectFilter(record); + + let outputValidator: LookupByPath | undefined; + + if (operationName) { + const operationSettings: Readonly | undefined = + record.projectConfig?.operationSettingsByOperationName.get(operationName); + + const outputFolderNames: string[] | undefined = operationSettings?.outputFolderNames; + if (outputFolderNames) { + const { relativePrefix } = record; + outputValidator = new LookupByPath(); + for (const folderName of outputFolderNames) { + outputValidator.setItem(`${relativePrefix}/${folderName}`, folderName); + } + } + + // Hash any additional files (files outside of a project, untracked project files, or even files outside of the repository) + const additionalFilesForOperation: ReadonlySet | undefined = + record.additionalFilesByOperationName?.get(operationName); + if (additionalFilesForOperation) { + for (const [filePath, hash] of this._resolveHashes(additionalFilesForOperation)) { + hashes.set(filePath, hash); + } + } + } + + const { _globalAdditionalHashes: globalAdditionalHashes } = this; + if (globalAdditionalHashes) { + for (const [file, hash] of globalAdditionalHashes) { + record.hashes.set(file, hash); + } + } + + // Hash the base project files + for (const [filePath, hash] of record.hashes) { + if (filter(filePath)) { + hashes.set(filePath, hash); + } + + // Ensure that the configured output folders for this operation do not contain any input files + // This should be reworked to operate on a global file origin map to ensure a hashed input + // is not a declared output of *any* operation. + const outputMatch: string | undefined = outputValidator?.findChildPath(filePath); + if (outputMatch) { + throw new Error( + `Configured output folder "${outputMatch}" for operation "${operationName}" in project "${project.projectRelativeFolder}" contains tracked input file "${filePath}".` + + ` If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from "outputFolderNames".` + + ` This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder.` + ); + } + } + } + + return hashes; + } + + /** + * {@inheritdoc} + */ + public getOperationOwnStateHash( + project: IRushConfigurationProjectForSnapshot, + operationName?: string + ): string { + const record: IInternalRushSnapshotProjectMetadata | undefined = this._projectMetadataMap.get(project); + if (!record) { + throw new Error(`No information available for project at ${project.projectFolder}`); + } + + const { hashByOperationName } = record; + let hash: string | undefined = hashByOperationName.get(operationName); + if (!hash) { + const hashes: ReadonlyMap = this.getTrackedFileHashesForOperation( + project, + operationName + ); + + const hasher: Hash = createHash('sha1'); + // If this is for a specific operation, apply operation-specific options + if (operationName) { + const operationSettings: Readonly | undefined = + record.projectConfig?.operationSettingsByOperationName.get(operationName); + if (operationSettings) { + const { dependsOnEnvVars, outputFolderNames } = operationSettings; + if (dependsOnEnvVars) { + // As long as we enumerate environment variables in a consistent order, we will get a stable hash. + // Changing the order in rush-project.json will change the hash anyway since the file contents are part of the hash. + for (const envVar of dependsOnEnvVars) { + hasher.update(`${hashDelimiter}$${envVar}=${this._environment[envVar] || ''}`); + } + } + + if (outputFolderNames) { + hasher.update(`${hashDelimiter}${JSON.stringify(outputFolderNames)}`); + } + } + } + + // Hash the base project files + for (const [filePath, fileHash] of hashes) { + hasher.update(`${hashDelimiter}${filePath}${hashDelimiter}${fileHash}`); + } + + hash = hasher.digest('hex'); + + hashByOperationName.set(operationName, hash); + } + + return hash; + } + + private *_resolveHashes(filePaths: Iterable): Generator<[string, string]> { + const { hashes, _additionalHashes } = this; + + for (const filePath of filePaths) { + const hash: string | undefined = hashes.get(filePath) ?? _additionalHashes?.get(filePath); + if (!hash) { + throw new Error(`Could not find hash for file path "${filePath}"`); + } + yield [filePath, hash]; + } + } + + private _getOrCreateProjectFilter( + record: IInternalRushSnapshotProjectMetadata + ): (filePath: string) => boolean { + if (!record.projectFilePathFilter) { + const ignoredGlobs: readonly string[] | undefined = record.projectConfig?.incrementalBuildIgnoredGlobs; + if (!ignoredGlobs || ignoredGlobs.length === 0) { + record.projectFilePathFilter = noopFilter; + } else { + const ignorer: Ignore = ignore(); + ignorer.add(ignoredGlobs as string[]); + const prefixLength: number = record.relativePrefix.length + 1; + record.projectFilePathFilter = function projectFilePathFilter(filePath: string): boolean { + return !ignorer.ignores(filePath.slice(prefixLength)); + }; + } + } + + return record.projectFilePathFilter; + } +} + +function getRelativePrefix(project: IRushConfigurationProjectForSnapshot, rootDir: string): string { + return Path.convertToSlashes(path.relative(rootDir, project.projectFolder)); +} + +function noopFilter(filePath: string): boolean { + return true; +} diff --git a/libraries/rush-lib/src/logic/snapshots/test/InputSnapshot.test.ts b/libraries/rush-lib/src/logic/snapshots/test/InputSnapshot.test.ts new file mode 100644 index 00000000000..b0b88dc6917 --- /dev/null +++ b/libraries/rush-lib/src/logic/snapshots/test/InputSnapshot.test.ts @@ -0,0 +1,469 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { LookupByPath } from '@rushstack/lookup-by-path'; + +import type { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; +import { + InputSnapshot, + type IRushSnapshotParameters, + type IRushConfigurationProjectForSnapshot +} from '../InputSnapshot'; + +describe(InputSnapshot.name, () => { + function getTestConfig(): { + project: IRushConfigurationProjectForSnapshot; + options: IRushSnapshotParameters; + } { + const project: IRushConfigurationProjectForSnapshot = { + projectFolder: '/root/a', + projectRelativeFolder: 'a' + }; + + return { + project, + options: { + rootDir: '/root', + additionalHashes: new Map([['/ext/config.json', 'hash4']]), + hashes: new Map([ + ['a/file1.js', 'hash1'], + ['a/file2.js', 'hash2'], + ['a/lib/file3.js', 'hash3'], + ['common/config/some-config.json', 'hash5'] + ]), + lookupByPath: new LookupByPath([[project.projectRelativeFolder, project]]), + projectMap: new Map() + } + }; + } + + function getTrivialSnapshot(): { + project: IRushConfigurationProjectForSnapshot; + input: InputSnapshot; + } { + const { project, options } = getTestConfig(); + + const input: InputSnapshot = new InputSnapshot(options); + + return { project, input }; + } + + describe(InputSnapshot.prototype.getTrackedFileHashesForOperation.name, () => { + it('Handles trivial input', () => { + const { project, input } = getTrivialSnapshot(); + + const result: ReadonlyMap = input.getTrackedFileHashesForOperation(project); + + expect(result).toMatchSnapshot(); + expect(result.size).toEqual(3); + expect(result.get('a/file1.js')).toEqual('hash1'); + expect(result.get('a/file2.js')).toEqual('hash2'); + expect(result.get('a/lib/file3.js')).toEqual('hash3'); + }); + + it('Detects outputFileNames collisions', () => { + const { project, options } = getTestConfig(); + + const projectConfig: Pick = { + operationSettingsByOperationName: new Map([ + [ + '_phase:build', + { + operationName: '_phase:build', + outputFolderNames: ['lib'] + } + ] + ]) + }; + + options.projectMap = new Map([ + [ + project, + { + projectConfig: projectConfig as RushProjectConfiguration + } + ] + ]); + + const input: InputSnapshot = new InputSnapshot(options); + + expect(() => + input.getTrackedFileHashesForOperation(project, '_phase:build') + ).toThrowErrorMatchingSnapshot(); + }); + + it('Respects additionalFilesByOperationName', () => { + const { project, options } = getTestConfig(); + + const projectConfig: Pick = { + operationSettingsByOperationName: new Map([ + [ + '_phase:build', + { + operationName: '_phase:build' + } + ] + ]) + }; + + const input: InputSnapshot = new InputSnapshot({ + ...options, + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig as RushProjectConfiguration, + additionalFilesByOperationName: new Map([['_phase:build', new Set(['/ext/config.json'])]]) + } + ] + ]) + }); + + const result: ReadonlyMap = input.getTrackedFileHashesForOperation( + project, + '_phase:build' + ); + + expect(result).toMatchSnapshot(); + expect(result.size).toEqual(4); + expect(result.get('a/file1.js')).toEqual('hash1'); + expect(result.get('a/file2.js')).toEqual('hash2'); + expect(result.get('a/lib/file3.js')).toEqual('hash3'); + expect(result.get('/ext/config.json')).toEqual('hash4'); + }); + + it('Respects globalAdditionalFiles', () => { + const { project, options } = getTestConfig(); + + const projectConfig: Pick = { + operationSettingsByOperationName: new Map([ + [ + '_phase:build', + { + operationName: '_phase:build' + } + ] + ]) + }; + + const input: InputSnapshot = new InputSnapshot({ + ...options, + globalAdditionalFiles: new Set(['common/config/some-config.json']), + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig as RushProjectConfiguration + } + ] + ]) + }); + + const result: ReadonlyMap = input.getTrackedFileHashesForOperation( + project, + '_phase:build' + ); + + expect(result).toMatchSnapshot(); + expect(result.size).toEqual(4); + expect(result.get('a/file1.js')).toEqual('hash1'); + expect(result.get('a/file2.js')).toEqual('hash2'); + expect(result.get('a/lib/file3.js')).toEqual('hash3'); + expect(result.get('common/config/some-config.json')).toEqual('hash5'); + }); + + it('Respects incrementalBuildIgnoredGlobs', () => { + const { project, options } = getTestConfig(); + + const projectConfig: Pick = { + incrementalBuildIgnoredGlobs: ['*2.js'] + }; + + const input: InputSnapshot = new InputSnapshot({ + ...options, + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig as RushProjectConfiguration + } + ] + ]) + }); + + const result: ReadonlyMap = input.getTrackedFileHashesForOperation(project); + + expect(result).toMatchSnapshot(); + expect(result.size).toEqual(2); + expect(result.get('a/file1.js')).toEqual('hash1'); + expect(result.get('a/lib/file3.js')).toEqual('hash3'); + }); + }); + + describe(InputSnapshot.prototype.getOperationOwnStateHash.name, () => { + it('Handles trivial input', () => { + const { project, input } = getTrivialSnapshot(); + + const result: string = input.getOperationOwnStateHash(project); + + expect(result).toMatchSnapshot(); + }); + + it('Is invariant to input hash order', () => { + const { project, options } = getTestConfig(); + + const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project); + + const input: InputSnapshot = new InputSnapshot({ + ...options, + hashes: new Map(Array.from(options.hashes).reverse()) + }); + + const result: string = input.getOperationOwnStateHash(project); + + expect(result).toEqual(baseline); + }); + + it('Detects outputFileNames collisions', () => { + const { project, options } = getTestConfig(); + + const projectConfig: Pick = { + operationSettingsByOperationName: new Map([ + [ + '_phase:build', + { + operationName: '_phase:build', + outputFolderNames: ['lib'] + } + ] + ]) + }; + + options.projectMap = new Map([ + [ + project, + { + projectConfig: projectConfig as RushProjectConfiguration + } + ] + ]); + + const input: InputSnapshot = new InputSnapshot(options); + + expect(() => input.getOperationOwnStateHash(project, '_phase:build')).toThrowErrorMatchingSnapshot(); + }); + + it('Changes if outputFileNames changes', () => { + const { project, options } = getTestConfig(); + const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); + + const projectConfig1: Pick = { + operationSettingsByOperationName: new Map([ + [ + '_phase:build', + { + operationName: '_phase:build', + outputFolderNames: ['lib-commonjs'] + } + ] + ]) + }; + + const projectConfig2: Pick = { + operationSettingsByOperationName: new Map([ + [ + '_phase:build', + { + operationName: '_phase:build', + outputFolderNames: ['lib-esm'] + } + ] + ]) + }; + + const input1: InputSnapshot = new InputSnapshot({ + ...options, + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig1 as RushProjectConfiguration + } + ] + ]) + }); + + const input2: InputSnapshot = new InputSnapshot({ + ...options, + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig2 as RushProjectConfiguration + } + ] + ]) + }); + + const result1: string = input1.getOperationOwnStateHash(project, '_phase:build'); + + const result2: string = input2.getOperationOwnStateHash(project, '_phase:build'); + + expect(result1).not.toEqual(baseline); + expect(result2).not.toEqual(baseline); + expect(result1).not.toEqual(result2); + }); + + it('Respects additionalOutputFilesByOperationName', () => { + const { project, options } = getTestConfig(); + const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); + + const projectConfig: Pick = { + operationSettingsByOperationName: new Map([ + [ + '_phase:build', + { + operationName: '_phase:build' + } + ] + ]) + }; + + const input: InputSnapshot = new InputSnapshot({ + ...options, + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig as RushProjectConfiguration, + additionalFilesByOperationName: new Map([['_phase:build', new Set(['/ext/config.json'])]]) + } + ] + ]) + }); + + const result: string = input.getOperationOwnStateHash(project, '_phase:build'); + + expect(result).toMatchSnapshot(); + expect(result).not.toEqual(baseline); + }); + + it('Respects globalAdditionalFiles', () => { + const { project, options } = getTestConfig(); + const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); + + const input: InputSnapshot = new InputSnapshot({ + ...options, + globalAdditionalFiles: new Set(['common/config/some-config.json']) + }); + + const result: string = input.getOperationOwnStateHash(project); + + expect(result).toMatchSnapshot(); + expect(result).not.toEqual(baseline); + }); + + it('Respects incrementalBuildIgnoredGlobs', () => { + const { project, options } = getTestConfig(); + const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); + + const projectConfig1: Pick = { + incrementalBuildIgnoredGlobs: ['*2.js'] + }; + + const input1: InputSnapshot = new InputSnapshot({ + ...options, + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig1 as RushProjectConfiguration + } + ] + ]) + }); + + const result1: string = input1.getOperationOwnStateHash(project); + + expect(result1).toMatchSnapshot(); + expect(result1).not.toEqual(baseline); + + const projectConfig2: Pick = { + incrementalBuildIgnoredGlobs: ['*1.js'] + }; + + const input2: InputSnapshot = new InputSnapshot({ + ...options, + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig2 as RushProjectConfiguration + } + ] + ]) + }); + + const result2: string = input2.getOperationOwnStateHash(project); + + expect(result2).toMatchSnapshot(); + expect(result2).not.toEqual(baseline); + + expect(result2).not.toEqual(result1); + }); + + it('Respects dependsOnEnvVars', () => { + const { project, options } = getTestConfig(); + const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); + + const projectConfig1: Pick = { + operationSettingsByOperationName: new Map([ + [ + '_phase:build', + { + operationName: '_phase:build', + dependsOnEnvVars: ['ENV_VAR'] + } + ] + ]) + }; + + const input1: InputSnapshot = new InputSnapshot({ + ...options, + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig1 as RushProjectConfiguration + } + ] + ]), + environment: {} + }); + + const result1: string = input1.getOperationOwnStateHash(project, '_phase:build'); + + expect(result1).toMatchSnapshot(); + expect(result1).not.toEqual(baseline); + + const input2: InputSnapshot = new InputSnapshot({ + ...options, + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig1 as RushProjectConfiguration + } + ] + ]), + environment: { ENV_VAR: 'some_value' } + }); + + const result2: string = input2.getOperationOwnStateHash(project, '_phase:build'); + + expect(result2).toMatchSnapshot(); + expect(result2).not.toEqual(baseline); + expect(result2).not.toEqual(result1); + }); + }); +}); diff --git a/libraries/rush-lib/src/logic/snapshots/test/__snapshots__/InputSnapshot.test.ts.snap b/libraries/rush-lib/src/logic/snapshots/test/__snapshots__/InputSnapshot.test.ts.snap new file mode 100644 index 00000000000..3960d128ff8 --- /dev/null +++ b/libraries/rush-lib/src/logic/snapshots/test/__snapshots__/InputSnapshot.test.ts.snap @@ -0,0 +1,52 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`InputSnapshot getOperationOwnStateHash Detects outputFileNames collisions 1`] = `"Configured output folder \\"lib\\" for operation \\"_phase:build\\" in project \\"a\\" contains tracked input file \\"a/lib/file3.js\\". If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from \\"outputFolderNames\\". This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder."`; + +exports[`InputSnapshot getOperationOwnStateHash Handles trivial input 1`] = `"acf14e45ed255b0288e449432d48c285f619d146"`; + +exports[`InputSnapshot getOperationOwnStateHash Respects additionalOutputFilesByOperationName 1`] = `"07f4147294d21a07865de84875d60bfbcc690652"`; + +exports[`InputSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 1`] = `"ad1f915d0ac6331c09febbbe1496c970a5401b73"`; + +exports[`InputSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 2`] = `"2c68d56fc9278b6495496070a6a992b929c37a83"`; + +exports[`InputSnapshot getOperationOwnStateHash Respects globalAdditionalFiles 1`] = `"0e0437ad1941bacd098b22da15dc673f86ca6003"`; + +exports[`InputSnapshot getOperationOwnStateHash Respects incrementalBuildIgnoredGlobs 1`] = `"f7b5af9ffdaa39831ed3374f28d0f7dccbee9c8d"`; + +exports[`InputSnapshot getOperationOwnStateHash Respects incrementalBuildIgnoredGlobs 2`] = `"24047d4271ebf8badc9403c9a21a09fb6db2fb9c"`; + +exports[`InputSnapshot getTrackedFileHashesForOperation Detects outputFileNames collisions 1`] = `"Configured output folder \\"lib\\" for operation \\"_phase:build\\" in project \\"a\\" contains tracked input file \\"a/lib/file3.js\\". If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from \\"outputFolderNames\\". This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder."`; + +exports[`InputSnapshot getTrackedFileHashesForOperation Handles trivial input 1`] = ` +Map { + "a/file1.js" => "hash1", + "a/file2.js" => "hash2", + "a/lib/file3.js" => "hash3", +} +`; + +exports[`InputSnapshot getTrackedFileHashesForOperation Respects additionalFilesByOperationName 1`] = ` +Map { + "/ext/config.json" => "hash4", + "a/file1.js" => "hash1", + "a/file2.js" => "hash2", + "a/lib/file3.js" => "hash3", +} +`; + +exports[`InputSnapshot getTrackedFileHashesForOperation Respects globalAdditionalFiles 1`] = ` +Map { + "a/file1.js" => "hash1", + "a/file2.js" => "hash2", + "a/lib/file3.js" => "hash3", + "common/config/some-config.json" => "hash5", +} +`; + +exports[`InputSnapshot getTrackedFileHashesForOperation Respects incrementalBuildIgnoredGlobs 1`] = ` +Map { + "a/file1.js" => "hash1", + "a/lib/file3.js" => "hash3", +} +`; diff --git a/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts b/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts index 473b81c5444..63aab041323 100644 --- a/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts +++ b/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts @@ -1,322 +1,99 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { LookupByPath } from '@rushstack/lookup-by-path'; -import { StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; - -import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; -import type { RushConfiguration } from '../../api/RushConfiguration'; -import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; -import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; -import { UNINITIALIZED } from '../../utilities/Utilities'; - -describe(ProjectChangeAnalyzer.name, () => { - beforeEach(() => { - jest.spyOn(EnvironmentConfiguration, 'gitBinaryPath', 'get').mockReturnValue(undefined); - jest.spyOn(RushProjectConfiguration, 'tryLoadIgnoreGlobsForProjectAsync').mockResolvedValue(undefined); - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - function createTestSubject( - projects: RushConfigurationProject[], - files: Map - ): ProjectChangeAnalyzer { - const rushConfiguration: RushConfiguration = { - commonRushConfigFolder: '', - projects, - rushJsonFolder: '', - defaultSubspace: { - getCommittedShrinkwrapFilePath(variant: string | undefined): string { - return 'common/config/rush/pnpm-lock.yaml'; - } - }, - getProjectLookupForRoot(root: string): LookupByPath { - const lookup: LookupByPath = new LookupByPath(); - for (const project of projects) { - lookup.setItem(project.projectRelativeFolder, project); - } - return lookup; - }, - getProjectByName(name: string): RushConfigurationProject | undefined { - return projects.find((project) => project.packageName === name); - }, - getCurrentlyInstalledVariantAsync: () => Promise.resolve(undefined) - } as RushConfiguration; - - const subject: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(rushConfiguration); - - subject['_getRepoDepsAsync'] = jest.fn(() => { - return Promise.resolve({ - gitPath: 'git', - hashes: files, - rootDir: '' - }); - }); - - return subject; - } - - describe(ProjectChangeAnalyzer.prototype._tryGetProjectDependenciesAsync.name, () => { - it('returns the files for the specified project', async () => { - const projects: RushConfigurationProject[] = [ - { - packageName: 'apple', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/apple' - } as RushConfigurationProject, - { - packageName: 'banana', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/banana' - } as RushConfigurationProject - ]; - const files: Map = new Map([ - ['apps/apple/core.js', 'a101'], - ['apps/banana/peel.js', 'b201'] - ]); - const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); - const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( - new Map([['apps/apple/core.js', 'a101']]) - ); - expect(await subject._tryGetProjectDependenciesAsync(projects[1], terminal)).toEqual( - new Map([['apps/banana/peel.js', 'b201']]) - ); - }); - - it('ignores files specified by project configuration files, relative to project folder', async () => { - // rush-project.json configuration for 'apple' - jest - .spyOn(RushProjectConfiguration, 'tryLoadIgnoreGlobsForProjectAsync') - .mockResolvedValueOnce(['assets/*.png', '*.js.map']); - // rush-project.json configuration for 'banana' does not exist - jest - .spyOn(RushProjectConfiguration, 'tryLoadIgnoreGlobsForProjectAsync') - .mockResolvedValueOnce(undefined); - - const projects: RushConfigurationProject[] = [ - { - packageName: 'apple', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/apple' - } as RushConfigurationProject, - { - packageName: 'banana', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/banana' - } as RushConfigurationProject - ]; - const files: Map = new Map([ - ['apps/apple/core.js', 'a101'], - ['apps/apple/core.js.map', 'a102'], - ['apps/apple/assets/one.jpg', 'a103'], - ['apps/apple/assets/two.png', 'a104'], - ['apps/banana/peel.js', 'b201'], - ['apps/banana/peel.js.map', 'b202'] - ]); - const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); - const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( - new Map([ - ['apps/apple/core.js', 'a101'], - ['apps/apple/assets/one.jpg', 'a103'] - ]) - ); - expect(await subject._tryGetProjectDependenciesAsync(projects[1], terminal)).toEqual( - new Map([ - ['apps/banana/peel.js', 'b201'], - ['apps/banana/peel.js.map', 'b202'] - ]) - ); - }); - - it('interprets ignored globs as a dot-ignore file (not as individually handled globs)', async () => { - // rush-project.json configuration for 'apple' - jest - .spyOn(RushProjectConfiguration, 'tryLoadIgnoreGlobsForProjectAsync') - .mockResolvedValue(['*.png', 'assets/*.psd', '!assets/important/**']); - - const projects: RushConfigurationProject[] = [ - { - packageName: 'apple', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/apple' - } as RushConfigurationProject - ]; - const files: Map = new Map([ - ['apps/apple/one.png', 'a101'], - ['apps/apple/assets/two.psd', 'a102'], - ['apps/apple/assets/three.png', 'a103'], - ['apps/apple/assets/important/four.png', 'a104'], - ['apps/apple/assets/important/five.psd', 'a105'], - ['apps/apple/src/index.ts', 'a106'] - ]); - const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); - const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - - // In a dot-ignore file, the later rule '!assets/important/**' should override the previous - // rule of '*.png'. This unit test verifies that this behavior doesn't change later if - // we modify the implementation. - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( - new Map([ - ['apps/apple/assets/important/four.png', 'a104'], - ['apps/apple/assets/important/five.psd', 'a105'], - ['apps/apple/src/index.ts', 'a106'] - ]) - ); - }); +const mockHashes: Map = new Map([ + ['a/package.json', 'hash1'], + ['b/package.json', 'hash2'], + ['c/package.json', 'hash3'], + ['changes/a.json', 'hash4'], + ['changes/b.json', 'hash5'], + ['changes/c.json', 'hash6'], + ['changes/d.json', 'hash7'], + ['changes/h.json', 'hash8'], + ['common/config/rush/version-policies.json', 'hash9'], + ['common/config/rush/npm-shrinkwrap.json', 'hash10'], + ['d/package.json', 'hash11'], + ['e/package.json', 'hash12'], + ['f/package.json', 'hash13'], + ['g/package.json', 'hash14'], + ['h/package.json', 'hash15'], + ['i/package.json', 'hash16'], + ['j/package.json', 'hash17'], + ['rush.json', 'hash18'] +]); +jest.mock(`@rushstack/package-deps-hash`, () => { + return { + getRepoRoot(dir: string): string { + return dir; + }, + getRepoStateAsync(): ReadonlyMap { + return mockHashes; + }, + getRepoChangesAsync(): ReadonlyMap { + return new Map(); + }, + getGitHashForFiles(filePaths: Iterable): ReadonlyMap { + return new Map(Array.from(filePaths, (filePath: string) => [filePath, filePath])); + }, + hashFilesAsync(rootDirectory: string, filePaths: Iterable): ReadonlyMap { + return new Map(Array.from(filePaths, (filePath: string) => [filePath, filePath])); + } + }; +}); - it('includes the committed shrinkwrap file as a dep for all projects', async () => { - const projects: RushConfigurationProject[] = [ - { - packageName: 'apple', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/apple' - } as RushConfigurationProject, - { - packageName: 'banana', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/banana' - } as RushConfigurationProject - ]; - const files: Map = new Map([ - ['apps/apple/core.js', 'a101'], - ['apps/banana/peel.js', 'b201'], - ['common/config/rush/pnpm-lock.yaml', 'ffff'], - ['tools/random-file.js', 'e00e'] - ]); - const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); - const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); +const mockSnapshot: jest.Mock = jest.fn(); - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( - new Map([ - ['apps/apple/core.js', 'a101'], - ['common/config/rush/pnpm-lock.yaml', 'ffff'] - ]) - ); - expect(await subject._tryGetProjectDependenciesAsync(projects[1], terminal)).toEqual( - new Map([ - ['apps/banana/peel.js', 'b201'], - ['common/config/rush/pnpm-lock.yaml', 'ffff'] - ]) - ); - }); - - it('throws an exception if the specified project does not exist', async () => { - const projects: RushConfigurationProject[] = [ - { - packageName: 'apple', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/apple' - } as RushConfigurationProject - ]; - const files: Map = new Map([['apps/apple/core.js', 'a101']]); - const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); - const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); +jest.mock('../snapshots/InputSnapshot', () => { + return { + InputSnapshot: mockSnapshot + }; +}); - try { - await subject._tryGetProjectDependenciesAsync( - { - packageName: 'carrot' - } as RushConfigurationProject, - terminal - ); - fail('Should have thrown error'); - } catch (e) { - expect(e).toMatchSnapshot(); - } - }); +import { resolve } from 'node:path'; - it('lazy-loads project data and caches it for future calls', async () => { - const projects: RushConfigurationProject[] = [ - { - packageName: 'apple', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/apple' - } as RushConfigurationProject - ]; - const files: Map = new Map([['apps/apple/core.js', 'a101']]); - const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); - const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); +import { StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; - // Because other unit tests rely on the fact that a freshly instantiated - // ProjectChangeAnalyzer is inert until someone actually requests project data, - // this test makes that expectation explicit. +import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; +import { RushConfiguration } from '../../api/RushConfiguration'; +import type { + IInputSnapshot, + IInputSnapshotProvider, + IRushSnapshotParameters +} from '../snapshots/InputSnapshot'; - expect(subject['_data']).toEqual(UNINITIALIZED); - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( - new Map([['apps/apple/core.js', 'a101']]) - ); - expect(subject['_data']).toBeDefined(); - expect(subject['_data']).not.toEqual(UNINITIALIZED); - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( - new Map([['apps/apple/core.js', 'a101']]) - ); - expect(subject['_getRepoDepsAsync']).toHaveBeenCalledTimes(1); - }); +describe(ProjectChangeAnalyzer.name, () => { + beforeEach(() => { + mockSnapshot.mockClear(); }); - describe(ProjectChangeAnalyzer.prototype._tryGetProjectStateHashAsync.name, () => { - it('returns a fixed hash snapshot for a set of project deps', async () => { - const projects: RushConfigurationProject[] = [ - { - packageName: 'apple', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/apple' - } as RushConfigurationProject - ]; - const files: Map = new Map([ - ['apps/apple/core.js', 'a101'], - ['apps/apple/juice.js', 'e333'], - ['apps/apple/slices.js', 'a102'] - ]); - const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); - const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - - expect(await subject._tryGetProjectStateHashAsync(projects[0], terminal)).toMatchInlineSnapshot( - `"265536e325cdfac3fa806a51873d927a712fc6c9"` - ); - }); - - it('returns the same hash regardless of dep order', async () => { - const projectsA: RushConfigurationProject[] = [ - { - packageName: 'apple', - projectFolder: '/apps/apple', - projectRelativeFolder: 'apps/apple' - } as RushConfigurationProject - ]; - const filesA: Map = new Map([ - ['apps/apple/core.js', 'a101'], - ['apps/apple/juice.js', 'e333'], - ['apps/apple/slices.js', 'a102'] - ]); - const subjectA: ProjectChangeAnalyzer = createTestSubject(projectsA, filesA); - - const projectsB: RushConfigurationProject[] = [ - { - packageName: 'apple', - projectFolder: 'apps/apple', - projectRelativeFolder: 'apps/apple' - } as RushConfigurationProject - ]; - const filesB: Map = new Map([ - ['apps/apple/slices.js', 'a102'], - ['apps/apple/core.js', 'a101'], - ['apps/apple/juice.js', 'e333'] - ]); - const subjectB: ProjectChangeAnalyzer = createTestSubject(projectsB, filesB); - - const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - expect(await subjectA._tryGetProjectStateHashAsync(projectsA[0], terminal)).toEqual( - await subjectB._tryGetProjectStateHashAsync(projectsB[0], terminal) + describe(ProjectChangeAnalyzer.prototype._tryGetSnapshotProviderAsync.name, () => { + it('returns a snapshot', async () => { + const rootDir: string = resolve(__dirname, 'repo'); + const rushConfiguration: RushConfiguration = RushConfiguration.loadFromConfigurationFile( + resolve(rootDir, 'rush.json') ); + const projectChangeAnalyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(rushConfiguration); + const terminalProvider: StringBufferTerminalProvider = new StringBufferTerminalProvider(true); + const terminal: Terminal = new Terminal(terminalProvider); + const mockSnapshotValue: {} = {}; + mockSnapshot.mockImplementation(() => mockSnapshotValue); + const snapshotProvider: IInputSnapshotProvider | undefined = + await projectChangeAnalyzer._tryGetSnapshotProviderAsync(new Map(), terminal); + const snapshot: IInputSnapshot | undefined = await snapshotProvider?.(); + + expect(snapshot).toBe(mockSnapshotValue); + expect(terminalProvider.getErrorOutput()).toEqual(''); + expect(terminalProvider.getWarningOutput()).toEqual(''); + + expect(mockSnapshot).toHaveBeenCalledTimes(1); + + const mockInput: IRushSnapshotParameters = mockSnapshot.mock.calls[0][0]; + expect(mockInput.globalAdditionalFiles).toBeDefined(); + expect(mockInput.globalAdditionalFiles).toMatchObject(['common/config/rush/npm-shrinkwrap.json']); + + expect(mockInput.hashes).toEqual(mockHashes); + expect(mockInput.rootDir).toEqual(rootDir); + expect(mockInput.additionalHashes).toEqual(new Map()); }); }); }); diff --git a/libraries/rush-lib/src/logic/test/__snapshots__/ProjectChangeAnalyzer.test.ts.snap b/libraries/rush-lib/src/logic/test/__snapshots__/ProjectChangeAnalyzer.test.ts.snap deleted file mode 100644 index bc265de44ee..00000000000 --- a/libraries/rush-lib/src/logic/test/__snapshots__/ProjectChangeAnalyzer.test.ts.snap +++ /dev/null @@ -1,3 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`ProjectChangeAnalyzer _tryGetProjectDependenciesAsync throws an exception if the specified project does not exist 1`] = `[Error: Project "carrot" does not exist in the current Rush configuration.]`; diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 3e6762f74e0..01dec30d655 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -15,7 +15,6 @@ import type { IPhase } from '../api/CommandLineConfiguration'; import type { RushConfiguration } from '../api/RushConfiguration'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; import type { Operation } from '../logic/operations/Operation'; -import type { ProjectChangeAnalyzer } from '../logic/ProjectChangeAnalyzer'; import type { IExecutionResult, IOperationExecutionResult @@ -25,6 +24,7 @@ import type { RushProjectConfiguration } from '../api/RushProjectConfiguration'; import type { IOperationRunnerContext } from '../logic/operations/IOperationRunner'; import type { ITelemetryData } from '../logic/Telemetry'; import type { OperationStatus } from '../logic/operations/OperationStatus'; +import type { IInputSnapshot } from '../logic/snapshots/InputSnapshot'; /** * A plugin that interacts with a phased commands. @@ -77,7 +77,6 @@ export interface ICreateOperationsContext { * The set of phases selected for the current command execution. */ readonly phaseSelection: ReadonlySet; - /** * The set of Rush projects selected for the current command execution. */ @@ -109,11 +108,10 @@ export interface ICreateOperationsContext { */ export interface IExecuteOperationsContext extends ICreateOperationsContext { /** - * The current state of the repository. - * - * Note that this is not defined during the initial operation creation. + * The current state of the repository, if available. + * Not part of the creation context to avoid the overhead of Git calls when initializing the graph. */ - readonly projectChangeAnalyzer: ProjectChangeAnalyzer; + readonly inputSnapshot?: IInputSnapshot; } /** From ab87818d0c9bbd2aeccc39b2b567b92f3491c879 Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 30 Sep 2024 21:00:35 +0000 Subject: [PATCH 2/6] Rename InputSnapshot and related functionality --- ...ject-change-analyzer_2024-01-09-22-11.json | 2 +- common/reviews/api/rush-lib.api.md | 14 +-- .../cli/scriptActions/PhasedScriptAction.ts | 19 +-- libraries/rush-lib/src/index.ts | 6 +- .../src/logic/ProjectChangeAnalyzer.ts | 20 +-- .../rush-lib/src/logic/ProjectWatcher.ts | 36 +++--- .../buildCache/test/ProjectBuildCache.test.ts | 8 +- .../InputsSnapshot.ts} | 118 +++++++++--------- .../test/InputsSnapshot.test.ts} | 58 ++++----- .../__snapshots__/InputsSnapshot.test.ts.snap | 52 ++++++++ .../src/logic/operations/BuildPlanPlugin.ts | 4 +- .../operations/CacheableOperationPlugin.ts | 15 ++- .../src/logic/operations/LegacySkipPlugin.ts | 4 +- .../operations/test/BuildPlanPlugin.test.ts | 8 +- .../__snapshots__/InputSnapshot.test.ts.snap | 52 -------- .../logic/test/ProjectChangeAnalyzer.test.ts | 18 +-- .../src/pluginFramework/PhasedCommandHooks.ts | 4 +- 17 files changed, 221 insertions(+), 217 deletions(-) rename libraries/rush-lib/src/logic/{snapshots/InputSnapshot.ts => incremental/InputsSnapshot.ts} (83%) rename libraries/rush-lib/src/logic/{snapshots/test/InputSnapshot.test.ts => incremental/test/InputsSnapshot.test.ts} (86%) create mode 100644 libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap delete mode 100644 libraries/rush-lib/src/logic/snapshots/test/__snapshots__/InputSnapshot.test.ts.snap diff --git a/common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-11.json b/common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-11.json index ff408bd0f32..6da4b33c943 100644 --- a/common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-11.json +++ b/common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-11.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@microsoft/rush", - "comment": "(BREAKING CHANGE) Replace use of `ProjectChangeAnalyzer` in phased command hooks with a new `InputSnapshot` data structure that is completely synchronous and does not perform any disk operations. Perform all disk operations and state computation prior to executing the build graph.", + "comment": "(BREAKING CHANGE) Replace use of `ProjectChangeAnalyzer` in phased command hooks with a new `InputsSnapshot` data structure that is completely synchronous and does not perform any disk operations. Perform all disk operations and state computation prior to executing the build graph.", "type": "none" } ], diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 154718c5471..81df579ecf4 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -330,6 +330,9 @@ export class _FlagFile { // @beta export type GetCacheEntryIdFunction = (options: IGenerateCacheEntryIdOptions) => string; +// @beta +export type GetInputsSnapshotAsyncFn = () => Promise; + // @internal (undocumented) export interface _IBuiltInPluginConfiguration extends _IRushPluginConfigurationBase { // (undocumented) @@ -462,7 +465,7 @@ export interface IEnvironmentConfigurationInitializeOptions { // @alpha export interface IExecuteOperationsContext extends ICreateOperationsContext { - readonly inputSnapshot?: IInputSnapshot; + readonly inputsSnapshot?: IInputsSnapshot; } // @alpha @@ -522,18 +525,13 @@ export interface IGlobalCommand extends IRushCommand { } // @beta -export interface IInputSnapshot { +export interface IInputsSnapshot { getOperationOwnStateHash(project: IRushConfigurationProjectForSnapshot, operationName?: string): string; getTrackedFileHashesForOperation(project: IRushConfigurationProjectForSnapshot, operationName?: string): ReadonlyMap; readonly hashes: ReadonlyMap; readonly rootDirectory: string; } -// @beta -export interface IInputSnapshotProvider { - (): Promise; -} - // @public export interface ILaunchOptions { alreadyReportedNodeTooNewError?: boolean; @@ -1124,7 +1122,7 @@ export class ProjectChangeAnalyzer { // (undocumented) protected getChangesByProject(lookup: LookupByPath, changedFiles: Map): Map>; // @internal - _tryGetSnapshotProviderAsync(projectConfigurations: ReadonlyMap, terminal: ITerminal): Promise; + _tryGetSnapshotProviderAsync(projectConfigurations: ReadonlyMap, terminal: ITerminal): Promise; } // @public diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 6e4d5fe311b..e5c0d6a5a40 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -44,7 +44,7 @@ import type { ITelemetryData, ITelemetryOperationResult } from '../../logic/Tele import { parseParallelism } from '../parsing/ParseParallelism'; import { CobuildConfiguration } from '../../api/CobuildConfiguration'; import { CacheableOperationPlugin } from '../../logic/operations/CacheableOperationPlugin'; -import type { IInputSnapshot, IInputSnapshotProvider } from '../../logic/snapshots/InputSnapshot'; +import type { IInputsSnapshot, GetInputsSnapshotAsyncFn } from '../../logic/incremental/InputsSnapshot'; import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { LegacySkipPlugin } from '../../logic/operations/LegacySkipPlugin'; import { ValidateOperationsPlugin } from '../../logic/operations/ValidateOperationsPlugin'; @@ -81,8 +81,8 @@ interface IInitialRunPhasesOptions { } interface IRunPhasesOptions extends IInitialRunPhasesOptions { - snapshotProvider: IInputSnapshotProvider | undefined; - initialSnapshot: IInputSnapshot | undefined; + snapshotProvider: GetInputsSnapshotAsyncFn | undefined; + initialSnapshot: IInputsSnapshot | undefined; executionManagerOptions: IOperationExecutionManagerOptions; } @@ -550,9 +550,9 @@ export class PhasedScriptAction extends BaseScriptAction { repoStateStopwatch.start(); const analyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); - const snapshotProvider: IInputSnapshotProvider | undefined = + const snapshotProvider: GetInputsSnapshotAsyncFn | undefined = await analyzer._tryGetSnapshotProviderAsync(projectConfigurations, terminal); - const initialSnapshot: IInputSnapshot | undefined = await snapshotProvider?.(); + const initialSnapshot: IInputsSnapshot | undefined = await snapshotProvider?.(); repoStateStopwatch.stop(); terminal.writeLine(`DONE (${repoStateStopwatch.toString()})`); @@ -565,7 +565,7 @@ export class PhasedScriptAction extends BaseScriptAction { const initialExecuteOperationsContext: IExecuteOperationsContext = { ...initialCreateOperationsContext, - inputSnapshot: initialSnapshot + inputsSnapshot: initialSnapshot }; const executionManagerOptions: IOperationExecutionManagerOptions = { @@ -693,7 +693,7 @@ export class PhasedScriptAction extends BaseScriptAction { ); const projectWatcher: typeof ProjectWatcher.prototype = new ProjectWatcher({ - snapshotProvider, + getInputSnapshotAsync: snapshotProvider, initialState, debounceMs: this._watchDebounceMs, rushConfiguration: this.rushConfiguration, @@ -731,7 +731,8 @@ export class PhasedScriptAction extends BaseScriptAction { // eslint-disable-next-line no-constant-condition while (true) { // On the initial invocation, this promise will return immediately with the full set of projects - const { changedProjects, state } = await projectWatcher.waitForChangeAsync(onWaitingForChanges); + const { changedProjects, inputsSnapshot: state } = + await projectWatcher.waitForChangeAsync(onWaitingForChanges); if (stopwatch.state === StopwatchState.Stopped) { // Clear and reset the stopwatch so that we only report time from a single execution at a time @@ -751,7 +752,7 @@ export class PhasedScriptAction extends BaseScriptAction { const executeOperationsContext: IExecuteOperationsContext = { ...initialCreateOperationsContext, isInitial: false, - inputSnapshot: state, + inputsSnapshot: state, projectsInUnknownState: changedProjects, phaseOriginal, phaseSelection, diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index fb7c1c35a42..e3c5e19c20e 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -123,10 +123,10 @@ export { export { ProjectChangeAnalyzer, type IGetChangedProjectsOptions } from './logic/ProjectChangeAnalyzer'; export type { - IInputSnapshot, - IInputSnapshotProvider, + IInputsSnapshot, + GetInputsSnapshotAsyncFn as GetInputsSnapshotAsyncFn, IRushConfigurationProjectForSnapshot -} from './logic/snapshots/InputSnapshot'; +} from './logic/incremental/InputsSnapshot'; export type { IOperationRunner, IOperationRunnerContext } from './logic/operations/IOperationRunner'; export type { diff --git a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts index fa739d22c12..17e699e21d3 100644 --- a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts +++ b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts @@ -22,11 +22,11 @@ import { BaseProjectShrinkwrapFile } from './base/BaseProjectShrinkwrapFile'; import { PnpmShrinkwrapFile } from './pnpm/PnpmShrinkwrapFile'; import { Git } from './Git'; import { - type IRushSnapshotProjectMetadata, - type IInputSnapshot, - InputSnapshot, - type IInputSnapshotProvider -} from './snapshots/InputSnapshot'; + type IInputsSnapshotProjectMetadata, + type IInputsSnapshot, + InputsSnapshot, + type GetInputsSnapshotAsyncFn +} from './incremental/InputsSnapshot'; /** * @beta @@ -202,7 +202,7 @@ export class ProjectChangeAnalyzer { public async _tryGetSnapshotProviderAsync( projectConfigurations: ReadonlyMap, terminal: ITerminal - ): Promise { + ): Promise { try { const gitPath: string = this._git.getGitPathOrThrow(); @@ -218,13 +218,13 @@ export class ProjectChangeAnalyzer { // Load the rush-project.json files for the whole repository const additionalGlobs: IAdditionalGlob[] = []; - const projectMap: Map = new Map(); + const projectMap: Map = new Map(); for (const project of rushConfiguration.projects) { const projectConfig: RushProjectConfiguration | undefined = projectConfigurations.get(project); const additionalFilesByOperationName: Map> = new Map(); - const projectMetadata: IRushSnapshotProjectMetadata = { + const projectMetadata: IInputsSnapshotProjectMetadata = { projectConfig, additionalFilesByOperationName }; @@ -291,7 +291,7 @@ export class ProjectChangeAnalyzer { const lookupByPath: IReadonlyLookupByPath = this._rushConfiguration.getProjectLookupForRoot(rootDirectory); - return async function tryGetSnapshotAsync(): Promise { + return async function tryGetSnapshotAsync(): Promise { try { const [hashes, additionalFiles] = await Promise.all([ getRepoStateAsync(rootDirectory, additionalRelativePathsToHash, gitPath), @@ -313,7 +313,7 @@ export class ProjectChangeAnalyzer { await hashFilesAsync(rootDirectory, additionalFiles, gitPath) ); - return new InputSnapshot({ + return new InputsSnapshot({ additionalHashes, globalAdditionalFiles, hashes, diff --git a/libraries/rush-lib/src/logic/ProjectWatcher.ts b/libraries/rush-lib/src/logic/ProjectWatcher.ts index 1d6af513ad9..a123726b1cd 100644 --- a/libraries/rush-lib/src/logic/ProjectWatcher.ts +++ b/libraries/rush-lib/src/logic/ProjectWatcher.ts @@ -10,17 +10,17 @@ import { AlreadyReportedError, Path, type FileSystemStats, FileSystem } from '@r import { Colorize, type ITerminal } from '@rushstack/terminal'; import { Git } from './Git'; -import type { IInputSnapshot, IInputSnapshotProvider } from './snapshots/InputSnapshot'; +import type { IInputsSnapshot, GetInputsSnapshotAsyncFn } from './incremental/InputsSnapshot'; import type { RushConfiguration } from '../api/RushConfiguration'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; export interface IProjectWatcherOptions { - snapshotProvider: IInputSnapshotProvider; + getInputSnapshotAsync: GetInputsSnapshotAsyncFn; debounceMs?: number; rushConfiguration: RushConfiguration; projectsToWatch: ReadonlySet; terminal: ITerminal; - initialState?: IInputSnapshot | undefined; + initialState?: IInputsSnapshot | undefined; } export interface IProjectChangeResult { @@ -31,7 +31,7 @@ export interface IProjectChangeResult { /** * Contains the git hashes for all tracked files in the repo */ - state: IInputSnapshot; + inputsSnapshot: IInputsSnapshot; } export interface IPromptGeneratorFunction { @@ -46,22 +46,22 @@ interface IPathWatchOptions { * This class is for incrementally watching a set of projects in the repository for changes. * * We are manually using fs.watch() instead of `chokidar` because all we want from the file system watcher is a boolean - * signal indicating that "at least 1 file in a watched project changed". We then defer to IInputSnapshotProvider (which + * signal indicating that "at least 1 file in a watched project changed". We then defer to getInputsSnapshotAsync (which * is responsible for change detection in all incremental builds) to determine what actually chanaged. * * Calling `waitForChange()` will return a promise that resolves when the package-deps of one or * more projects differ from the value the previous time it was invoked. The first time will always resolve with the full selection. */ export class ProjectWatcher { - private readonly _snapshotProvider: IInputSnapshotProvider; + private readonly _getInputsSnapshotAsync: GetInputsSnapshotAsyncFn; private readonly _debounceMs: number; private readonly _repoRoot: string; private readonly _rushConfiguration: RushConfiguration; private readonly _projectsToWatch: ReadonlySet; private readonly _terminal: ITerminal; - private _initialState: IInputSnapshot | undefined; - private _previousState: IInputSnapshot | undefined; + private _initialState: IInputsSnapshot | undefined; + private _previousState: IInputsSnapshot | undefined; private _forceChangedProjects: Map = new Map(); private _resolveIfChanged: undefined | (() => Promise); private _getPromptLines: undefined | IPromptGeneratorFunction; @@ -72,7 +72,7 @@ export class ProjectWatcher { public constructor(options: IProjectWatcherOptions) { const { - snapshotProvider, + getInputSnapshotAsync: snapshotProvider, debounceMs = 1000, rushConfiguration, projectsToWatch, @@ -93,7 +93,7 @@ export class ProjectWatcher { this._renderedStatusLines = 0; this._getPromptLines = undefined; - this._snapshotProvider = snapshotProvider; + this._getInputsSnapshotAsync = snapshotProvider; } public pause(): void { @@ -143,7 +143,7 @@ export class ProjectWatcher { public async waitForChangeAsync(onWatchingFiles?: () => void): Promise { const initialChangeResult: IProjectChangeResult = await this._computeChangedAsync(); // Ensure that the new state is recorded so that we don't loop infinitely - this._commitChanges(initialChangeResult.state); + this._commitChanges(initialChangeResult.inputsSnapshot); if (initialChangeResult.changedProjects.size) { // We can't call `clear()` here due to the async tick in the end of _computeChanged for (const project of initialChangeResult.changedProjects) { @@ -155,7 +155,7 @@ export class ProjectWatcher { return initialChangeResult; } - const previousState: IInputSnapshot = initialChangeResult.state; + const previousState: IInputsSnapshot = initialChangeResult.inputsSnapshot; const repoRoot: string = Path.convertToSlashes(this._rushConfiguration.rushJsonFolder); // Map of path to whether config for the path @@ -238,7 +238,7 @@ export class ProjectWatcher { } } - this._commitChanges(result.state); + this._commitChanges(result.inputsSnapshot); const hasForcedChanges: boolean = this._forceChangedProjects.size > 0; if (hasForcedChanges) { @@ -400,18 +400,18 @@ export class ProjectWatcher { * Determines which, if any, projects (within the selection) have new hashes for files that are not in .gitignore */ private async _computeChangedAsync(): Promise { - const state: IInputSnapshot | undefined = await this._snapshotProvider(); + const state: IInputsSnapshot | undefined = await this._getInputsSnapshotAsync(); if (!state) { throw new AlreadyReportedError(); } - const previousState: IInputSnapshot | undefined = this._previousState; + const previousState: IInputsSnapshot | undefined = this._previousState; if (!previousState) { return { changedProjects: this._projectsToWatch, - state + inputsSnapshot: state }; } @@ -434,11 +434,11 @@ export class ProjectWatcher { return { changedProjects, - state + inputsSnapshot: state }; } - private _commitChanges(state: IInputSnapshot): void { + private _commitChanges(state: IInputsSnapshot): void { this._previousState = state; if (!this._initialState) { this._initialState = state; diff --git a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts index 1ba67061f54..1acc025dee4 100644 --- a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts +++ b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts @@ -36,7 +36,9 @@ describe(ProjectBuildCache.name, () => { projectRelativeFolder: 'apps/acme-wizard', dependencyProjects: [] } as unknown as RushConfigurationProject, - operationStateHash: 'build', + // Value from past tests, for consistency. + // The project build cache is not responsible for calculating this value. + operationStateHash: '1926f30e8ed24cb47be89aea39e7efd70fcda075', terminal, phaseName: 'build' }); @@ -47,7 +49,9 @@ describe(ProjectBuildCache.name, () => { describe(ProjectBuildCache.getProjectBuildCache.name, () => { it('returns a ProjectBuildCache with a calculated cacheId value', () => { const subject: ProjectBuildCache = prepareSubject({}); - expect(subject['_cacheId']).toMatchInlineSnapshot(`"acme-wizard/build"`); + expect(subject['_cacheId']).toMatchInlineSnapshot( + `"acme-wizard/1926f30e8ed24cb47be89aea39e7efd70fcda075"` + ); }); }); }); diff --git a/libraries/rush-lib/src/logic/snapshots/InputSnapshot.ts b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts similarity index 83% rename from libraries/rush-lib/src/logic/snapshots/InputSnapshot.ts rename to libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts index bfc0463cb8c..94c31ea19c3 100644 --- a/libraries/rush-lib/src/logic/snapshots/InputSnapshot.ts +++ b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts @@ -3,6 +3,7 @@ import * as path from 'node:path'; import { createHash, type Hash } from 'node:crypto'; + import ignore, { type Ignore } from 'ignore'; import { type IReadonlyLookupByPath, LookupByPath } from '@rushstack/lookup-by-path'; @@ -23,7 +24,7 @@ export type IRushConfigurationProjectForSnapshot = Pick< /** * @internal */ -export interface IRushSnapshotProjectMetadata { +export interface IInputsSnapshotProjectMetadata { /** * The contents of rush-project.json for the project, if available */ @@ -34,7 +35,7 @@ export interface IRushSnapshotProjectMetadata { additionalFilesByOperationName?: ReadonlyMap>; } -interface IInternalRushSnapshotProjectMetadata extends IRushSnapshotProjectMetadata { +interface IInternalInputsSnapshotProjectMetadata extends IInputsSnapshotProjectMetadata { /** * Cached filter of files that are not ignored by the project's `incrementalBuildIgnoredGlobs`. * @param filePath - The path to the file to check @@ -62,28 +63,23 @@ interface IInternalRushSnapshotProjectMetadata extends IRushSnapshotProjectMetad export type IRushSnapshotProjectMetadataMap = ReadonlyMap< IRushConfigurationProjectForSnapshot, - IRushSnapshotProjectMetadata + IInputsSnapshotProjectMetadata >; /** - * A function that can be invoked to get the current Rush snapshot. - * Binds the project configurations when created. + * Function that computes a new snapshot of the current state of the repository as of the current moment. + * Rush-level configuration state will have been bound during creation of the function. + * Captures the state of the environment, tracked files, and additional files. * * @beta */ -export interface IInputSnapshotProvider { - /** - * Compute a new snapshot of the current state of the repository as of the current moment. - * Captures the state of the environment, tracked files, and additional files. - */ - (): Promise; -} +export type GetInputsSnapshotAsyncFn = () => Promise; /** - * The parameters for constructing an {@link InputSnapshot}. + * The parameters for constructing an {@link InputsSnapshot}. * @internal */ -export interface IRushSnapshotParameters { +export interface IInputsSnapshotParameters { /** * Hashes for files selected by `dependsOnAdditionalFiles`. * Separated out to prevent being auto-assigned to a project. @@ -124,7 +120,7 @@ const { hashDelimiter } = RushConstants; * The methods on this interface are idempotent and will return the same result regardless of when they are executed. * @beta */ -export interface IInputSnapshot { +export interface IInputsSnapshot { /** * The raw hashes of all tracked files in the repository. */ @@ -167,13 +163,13 @@ export interface IInputSnapshot { * * @internal */ -export class InputSnapshot implements IInputSnapshot { +export class InputsSnapshot implements IInputsSnapshot { /** - * {@inheritdoc IInputSnapshot.hashes} + * {@inheritdoc IInputsSnapshot.hashes} */ public readonly hashes: ReadonlyMap; /** - * {@inheritdoc IInputSnapshot.rootDirectory} + * {@inheritdoc IInputsSnapshot.rootDirectory} */ public readonly rootDirectory: string; @@ -182,7 +178,7 @@ export class InputSnapshot implements IInputSnapshot { */ private readonly _projectMetadataMap: Map< IRushConfigurationProjectForSnapshot, - IInternalRushSnapshotProjectMetadata + IInternalInputsSnapshotProjectMetadata >; /** * Hashes of files to be included in all result sets. @@ -202,7 +198,7 @@ export class InputSnapshot implements IInputSnapshot { * @param params - The parameters for the snapshot * @internal */ - public constructor(params: IRushSnapshotParameters) { + public constructor(params: IInputsSnapshotParameters) { const { additionalHashes, environment = { ...process.env }, @@ -213,26 +209,10 @@ export class InputSnapshot implements IInputSnapshot { } = params; const projectMetadataMap: Map< IRushConfigurationProjectForSnapshot, - IInternalRushSnapshotProjectMetadata + IInternalInputsSnapshotProjectMetadata > = new Map(); - const createInternalRecord = ( - project: IRushConfigurationProjectForSnapshot, - baseRecord: IRushSnapshotProjectMetadata | undefined - ): IInternalRushSnapshotProjectMetadata => { - return { - // Data from the caller - projectConfig: baseRecord?.projectConfig, - additionalFilesByOperationName: baseRecord?.additionalFilesByOperationName, - - // Caches - hashes: new Map(), - hashByOperationName: new Map(), - fileHashesByOperationName: new Map(), - relativePrefix: getRelativePrefix(project, rootDir) - }; - }; for (const [project, record] of params.projectMap) { - projectMetadataMap.set(project, createInternalRecord(project, record)); + projectMetadataMap.set(project, createInternalRecord(project, record, rootDir)); } // Route hashes to individual projects @@ -242,9 +222,9 @@ export class InputSnapshot implements IInputSnapshot { continue; } - let record: IInternalRushSnapshotProjectMetadata | undefined = projectMetadataMap.get(project); + let record: IInternalInputsSnapshotProjectMetadata | undefined = projectMetadataMap.get(project); if (!record) { - projectMetadataMap.set(project, (record = createInternalRecord(project, undefined))); + projectMetadataMap.set(project, (record = createInternalRecord(project, undefined, rootDir))); } record.hashes.set(file, hash); @@ -291,7 +271,7 @@ export class InputSnapshot implements IInputSnapshot { project: IRushConfigurationProjectForSnapshot, operationName?: string ): ReadonlyMap { - const record: IInternalRushSnapshotProjectMetadata | undefined = this._projectMetadataMap.get(project); + const record: IInternalInputsSnapshotProjectMetadata | undefined = this._projectMetadataMap.get(project); if (!record) { throw new InternalError(`No information available for project at ${project.projectFolder}`); } @@ -302,7 +282,7 @@ export class InputSnapshot implements IInputSnapshot { hashes = new Map(); fileHashesByOperationName.set(operationName, hashes); // TODO: Support incrementalBuildIgnoredGlobs per-operation - const filter: (filePath: string) => boolean = this._getOrCreateProjectFilter(record); + const filter: (filePath: string) => boolean = getOrCreateProjectFilter(record); let outputValidator: LookupByPath | undefined; @@ -366,7 +346,7 @@ export class InputSnapshot implements IInputSnapshot { project: IRushConfigurationProjectForSnapshot, operationName?: string ): string { - const record: IInternalRushSnapshotProjectMetadata | undefined = this._projectMetadataMap.get(project); + const record: IInternalInputsSnapshotProjectMetadata | undefined = this._projectMetadataMap.get(project); if (!record) { throw new Error(`No information available for project at ${project.projectFolder}`); } @@ -424,26 +404,44 @@ export class InputSnapshot implements IInputSnapshot { yield [filePath, hash]; } } +} - private _getOrCreateProjectFilter( - record: IInternalRushSnapshotProjectMetadata - ): (filePath: string) => boolean { - if (!record.projectFilePathFilter) { - const ignoredGlobs: readonly string[] | undefined = record.projectConfig?.incrementalBuildIgnoredGlobs; - if (!ignoredGlobs || ignoredGlobs.length === 0) { - record.projectFilePathFilter = noopFilter; - } else { - const ignorer: Ignore = ignore(); - ignorer.add(ignoredGlobs as string[]); - const prefixLength: number = record.relativePrefix.length + 1; - record.projectFilePathFilter = function projectFilePathFilter(filePath: string): boolean { - return !ignorer.ignores(filePath.slice(prefixLength)); - }; - } +function getOrCreateProjectFilter( + record: IInternalInputsSnapshotProjectMetadata +): (filePath: string) => boolean { + if (!record.projectFilePathFilter) { + const ignoredGlobs: readonly string[] | undefined = record.projectConfig?.incrementalBuildIgnoredGlobs; + if (!ignoredGlobs || ignoredGlobs.length === 0) { + record.projectFilePathFilter = noopFilter; + } else { + const ignorer: Ignore = ignore(); + ignorer.add(ignoredGlobs as string[]); + const prefixLength: number = record.relativePrefix.length + 1; + record.projectFilePathFilter = function projectFilePathFilter(filePath: string): boolean { + return !ignorer.ignores(filePath.slice(prefixLength)); + }; } - - return record.projectFilePathFilter; } + + return record.projectFilePathFilter; +} + +function createInternalRecord( + project: IRushConfigurationProjectForSnapshot, + baseRecord: IInputsSnapshotProjectMetadata | undefined, + rootDir: string +): IInternalInputsSnapshotProjectMetadata { + return { + // Data from the caller + projectConfig: baseRecord?.projectConfig, + additionalFilesByOperationName: baseRecord?.additionalFilesByOperationName, + + // Caches + hashes: new Map(), + hashByOperationName: new Map(), + fileHashesByOperationName: new Map(), + relativePrefix: getRelativePrefix(project, rootDir) + }; } function getRelativePrefix(project: IRushConfigurationProjectForSnapshot, rootDir: string): string { diff --git a/libraries/rush-lib/src/logic/snapshots/test/InputSnapshot.test.ts b/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts similarity index 86% rename from libraries/rush-lib/src/logic/snapshots/test/InputSnapshot.test.ts rename to libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts index b0b88dc6917..a082fbc5098 100644 --- a/libraries/rush-lib/src/logic/snapshots/test/InputSnapshot.test.ts +++ b/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts @@ -5,15 +5,15 @@ import { LookupByPath } from '@rushstack/lookup-by-path'; import type { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; import { - InputSnapshot, - type IRushSnapshotParameters, + InputsSnapshot, + type IInputsSnapshotParameters, type IRushConfigurationProjectForSnapshot -} from '../InputSnapshot'; +} from '../InputsSnapshot'; -describe(InputSnapshot.name, () => { +describe(InputsSnapshot.name, () => { function getTestConfig(): { project: IRushConfigurationProjectForSnapshot; - options: IRushSnapshotParameters; + options: IInputsSnapshotParameters; } { const project: IRushConfigurationProjectForSnapshot = { projectFolder: '/root/a', @@ -39,16 +39,16 @@ describe(InputSnapshot.name, () => { function getTrivialSnapshot(): { project: IRushConfigurationProjectForSnapshot; - input: InputSnapshot; + input: InputsSnapshot; } { const { project, options } = getTestConfig(); - const input: InputSnapshot = new InputSnapshot(options); + const input: InputsSnapshot = new InputsSnapshot(options); return { project, input }; } - describe(InputSnapshot.prototype.getTrackedFileHashesForOperation.name, () => { + describe(InputsSnapshot.prototype.getTrackedFileHashesForOperation.name, () => { it('Handles trivial input', () => { const { project, input } = getTrivialSnapshot(); @@ -85,7 +85,7 @@ describe(InputSnapshot.name, () => { ] ]); - const input: InputSnapshot = new InputSnapshot(options); + const input: InputsSnapshot = new InputsSnapshot(options); expect(() => input.getTrackedFileHashesForOperation(project, '_phase:build') @@ -106,7 +106,7 @@ describe(InputSnapshot.name, () => { ]) }; - const input: InputSnapshot = new InputSnapshot({ + const input: InputsSnapshot = new InputsSnapshot({ ...options, projectMap: new Map([ [ @@ -146,7 +146,7 @@ describe(InputSnapshot.name, () => { ]) }; - const input: InputSnapshot = new InputSnapshot({ + const input: InputsSnapshot = new InputsSnapshot({ ...options, globalAdditionalFiles: new Set(['common/config/some-config.json']), projectMap: new Map([ @@ -179,7 +179,7 @@ describe(InputSnapshot.name, () => { incrementalBuildIgnoredGlobs: ['*2.js'] }; - const input: InputSnapshot = new InputSnapshot({ + const input: InputsSnapshot = new InputsSnapshot({ ...options, projectMap: new Map([ [ @@ -200,7 +200,7 @@ describe(InputSnapshot.name, () => { }); }); - describe(InputSnapshot.prototype.getOperationOwnStateHash.name, () => { + describe(InputsSnapshot.prototype.getOperationOwnStateHash.name, () => { it('Handles trivial input', () => { const { project, input } = getTrivialSnapshot(); @@ -212,9 +212,9 @@ describe(InputSnapshot.name, () => { it('Is invariant to input hash order', () => { const { project, options } = getTestConfig(); - const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project); + const baseline: string = new InputsSnapshot(options).getOperationOwnStateHash(project); - const input: InputSnapshot = new InputSnapshot({ + const input: InputsSnapshot = new InputsSnapshot({ ...options, hashes: new Map(Array.from(options.hashes).reverse()) }); @@ -248,14 +248,14 @@ describe(InputSnapshot.name, () => { ] ]); - const input: InputSnapshot = new InputSnapshot(options); + const input: InputsSnapshot = new InputsSnapshot(options); expect(() => input.getOperationOwnStateHash(project, '_phase:build')).toThrowErrorMatchingSnapshot(); }); it('Changes if outputFileNames changes', () => { const { project, options } = getTestConfig(); - const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); + const baseline: string = new InputsSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); const projectConfig1: Pick = { operationSettingsByOperationName: new Map([ @@ -281,7 +281,7 @@ describe(InputSnapshot.name, () => { ]) }; - const input1: InputSnapshot = new InputSnapshot({ + const input1: InputsSnapshot = new InputsSnapshot({ ...options, projectMap: new Map([ [ @@ -293,7 +293,7 @@ describe(InputSnapshot.name, () => { ]) }); - const input2: InputSnapshot = new InputSnapshot({ + const input2: InputsSnapshot = new InputsSnapshot({ ...options, projectMap: new Map([ [ @@ -316,7 +316,7 @@ describe(InputSnapshot.name, () => { it('Respects additionalOutputFilesByOperationName', () => { const { project, options } = getTestConfig(); - const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); + const baseline: string = new InputsSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); const projectConfig: Pick = { operationSettingsByOperationName: new Map([ @@ -329,7 +329,7 @@ describe(InputSnapshot.name, () => { ]) }; - const input: InputSnapshot = new InputSnapshot({ + const input: InputsSnapshot = new InputsSnapshot({ ...options, projectMap: new Map([ [ @@ -350,9 +350,9 @@ describe(InputSnapshot.name, () => { it('Respects globalAdditionalFiles', () => { const { project, options } = getTestConfig(); - const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); + const baseline: string = new InputsSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); - const input: InputSnapshot = new InputSnapshot({ + const input: InputsSnapshot = new InputsSnapshot({ ...options, globalAdditionalFiles: new Set(['common/config/some-config.json']) }); @@ -365,13 +365,13 @@ describe(InputSnapshot.name, () => { it('Respects incrementalBuildIgnoredGlobs', () => { const { project, options } = getTestConfig(); - const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); + const baseline: string = new InputsSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); const projectConfig1: Pick = { incrementalBuildIgnoredGlobs: ['*2.js'] }; - const input1: InputSnapshot = new InputSnapshot({ + const input1: InputsSnapshot = new InputsSnapshot({ ...options, projectMap: new Map([ [ @@ -392,7 +392,7 @@ describe(InputSnapshot.name, () => { incrementalBuildIgnoredGlobs: ['*1.js'] }; - const input2: InputSnapshot = new InputSnapshot({ + const input2: InputsSnapshot = new InputsSnapshot({ ...options, projectMap: new Map([ [ @@ -414,7 +414,7 @@ describe(InputSnapshot.name, () => { it('Respects dependsOnEnvVars', () => { const { project, options } = getTestConfig(); - const baseline: string = new InputSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); + const baseline: string = new InputsSnapshot(options).getOperationOwnStateHash(project, '_phase:build'); const projectConfig1: Pick = { operationSettingsByOperationName: new Map([ @@ -428,7 +428,7 @@ describe(InputSnapshot.name, () => { ]) }; - const input1: InputSnapshot = new InputSnapshot({ + const input1: InputsSnapshot = new InputsSnapshot({ ...options, projectMap: new Map([ [ @@ -446,7 +446,7 @@ describe(InputSnapshot.name, () => { expect(result1).toMatchSnapshot(); expect(result1).not.toEqual(baseline); - const input2: InputSnapshot = new InputSnapshot({ + const input2: InputsSnapshot = new InputsSnapshot({ ...options, projectMap: new Map([ [ diff --git a/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap b/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap new file mode 100644 index 00000000000..a1db7fa9818 --- /dev/null +++ b/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap @@ -0,0 +1,52 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`InputsSnapshot getOperationOwnStateHash Detects outputFileNames collisions 1`] = `"Configured output folder \\"lib\\" for operation \\"_phase:build\\" in project \\"a\\" contains tracked input file \\"a/lib/file3.js\\". If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from \\"outputFolderNames\\". This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder."`; + +exports[`InputsSnapshot getOperationOwnStateHash Handles trivial input 1`] = `"acf14e45ed255b0288e449432d48c285f619d146"`; + +exports[`InputsSnapshot getOperationOwnStateHash Respects additionalOutputFilesByOperationName 1`] = `"07f4147294d21a07865de84875d60bfbcc690652"`; + +exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 1`] = `"ad1f915d0ac6331c09febbbe1496c970a5401b73"`; + +exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 2`] = `"2c68d56fc9278b6495496070a6a992b929c37a83"`; + +exports[`InputsSnapshot getOperationOwnStateHash Respects globalAdditionalFiles 1`] = `"0e0437ad1941bacd098b22da15dc673f86ca6003"`; + +exports[`InputsSnapshot getOperationOwnStateHash Respects incrementalBuildIgnoredGlobs 1`] = `"f7b5af9ffdaa39831ed3374f28d0f7dccbee9c8d"`; + +exports[`InputsSnapshot getOperationOwnStateHash Respects incrementalBuildIgnoredGlobs 2`] = `"24047d4271ebf8badc9403c9a21a09fb6db2fb9c"`; + +exports[`InputsSnapshot getTrackedFileHashesForOperation Detects outputFileNames collisions 1`] = `"Configured output folder \\"lib\\" for operation \\"_phase:build\\" in project \\"a\\" contains tracked input file \\"a/lib/file3.js\\". If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from \\"outputFolderNames\\". This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder."`; + +exports[`InputsSnapshot getTrackedFileHashesForOperation Handles trivial input 1`] = ` +Map { + "a/file1.js" => "hash1", + "a/file2.js" => "hash2", + "a/lib/file3.js" => "hash3", +} +`; + +exports[`InputsSnapshot getTrackedFileHashesForOperation Respects additionalFilesByOperationName 1`] = ` +Map { + "/ext/config.json" => "hash4", + "a/file1.js" => "hash1", + "a/file2.js" => "hash2", + "a/lib/file3.js" => "hash3", +} +`; + +exports[`InputsSnapshot getTrackedFileHashesForOperation Respects globalAdditionalFiles 1`] = ` +Map { + "a/file1.js" => "hash1", + "a/file2.js" => "hash2", + "a/lib/file3.js" => "hash3", + "common/config/some-config.json" => "hash5", +} +`; + +exports[`InputsSnapshot getTrackedFileHashesForOperation Respects incrementalBuildIgnoredGlobs 1`] = ` +Map { + "a/file1.js" => "hash1", + "a/lib/file3.js" => "hash3", +} +`; diff --git a/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts b/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts index bc3f223c0b4..f48cd061e5d 100644 --- a/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts @@ -46,7 +46,7 @@ export class BuildPlanPlugin implements IPhasedCommandPlugin { recordByOperation: Map, context: IExecuteOperationsContext ): void { - const { projectConfigurations, inputSnapshot } = context; + const { projectConfigurations, inputsSnapshot } = context; const disjointSet: DisjointSet = new DisjointSet(); const operations: Operation[] = [...recordByOperation.keys()]; for (const operation of operations) { @@ -63,7 +63,7 @@ export class BuildPlanPlugin implements IPhasedCommandPlugin { const projectConfiguration: RushProjectConfiguration | undefined = projectConfigurations.get(associatedProject); const fileHashes: ReadonlyMap | undefined = - inputSnapshot?.getTrackedFileHashesForOperation(associatedProject, associatedPhase.name); + inputsSnapshot?.getTrackedFileHashesForOperation(associatedProject, associatedPhase.name); if (!fileHashes) { continue; } diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 6efbd7a1353..e680a7e0446 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -36,7 +36,7 @@ import type { OperationMetadataManager } from './OperationMetadataManager'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; import type { OperationExecutionRecord } from './OperationExecutionRecord'; -import type { IInputSnapshot } from '../snapshots/InputSnapshot'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; const PERIODIC_CALLBACK_INTERVAL_IN_SECONDS: number = 10; @@ -95,16 +95,16 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { recordByOperation: Map, context: IExecuteOperationsContext ): void => { - const { isIncrementalBuildAllowed, inputSnapshot, projectConfigurations, isInitial } = context; + const { isIncrementalBuildAllowed, inputsSnapshot, projectConfigurations, isInitial } = context; - if (!inputSnapshot) { + if (!inputsSnapshot) { throw new Error( `Build cache is only supported if running in a Git repository. Either disable the build cache or run Rush in a Git repository.` ); } // This redefinition is necessary due to limitations in TypeScript's control flow analysis, due to the nested closure. - const definitelyDefinedInputSnapshot: IInputSnapshot = inputSnapshot; + const definitelyDefinedInputsSnapshot: IInputsSnapshot = inputsSnapshot; const disjointSet: DisjointSet | undefined = cobuildConfiguration?.cobuildFeatureEnabled ? new DisjointSet() @@ -130,7 +130,10 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) const localStateHash: string | undefined = associatedProject && - definitelyDefinedInputSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name); + definitelyDefinedInputsSnapshot.getOperationOwnStateHash( + associatedProject, + associatedPhase?.name + ); // The final state hashes of operation dependencies are factored into the hash to ensure that any // state changes in dependencies will invalidate the cache. @@ -177,7 +180,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // This value can *currently* be cached per-project, but in the future the list of files will vary // depending on the selected phase. const fileHashes: ReadonlyMap | undefined = - inputSnapshot.getTrackedFileHashesForOperation(associatedProject, phaseName); + inputsSnapshot.getTrackedFileHashesForOperation(associatedProject, phaseName); const stateHash: string = getOrCreateOperationHash(operation); const cacheDisabledReason: string | undefined = projectConfiguration diff --git a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts index b6b0fcf68f8..651ec7849f8 100644 --- a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts @@ -73,7 +73,7 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { context: IExecuteOperationsContext ): void => { let logGitWarning: boolean = false; - const { inputSnapshot } = context; + const { inputsSnapshot } = context; for (const record of operations.values()) { const { operation } = record; @@ -102,7 +102,7 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { try { const fileHashes: ReadonlyMap | undefined = - inputSnapshot?.getTrackedFileHashesForOperation( + inputsSnapshot?.getTrackedFileHashesForOperation( associatedProject, operation.associatedPhase?.name ); diff --git a/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts index 3a7019c4070..d97de794e87 100644 --- a/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts @@ -24,7 +24,7 @@ import { RushConstants } from '../../RushConstants'; import { MockOperationRunner } from './MockOperationRunner'; import path from 'path'; import type { ICommandLineJson } from '../../../api/CommandLineJson'; -import type { IInputSnapshot } from '../../snapshots/InputSnapshot'; +import type { IInputsSnapshot } from '../../incremental/InputsSnapshot'; describe(BuildPlanPlugin.name, () => { const rushJsonFile: string = path.resolve(__dirname, `../../test/workspaceRepo/rush.json`); @@ -104,13 +104,13 @@ describe(BuildPlanPlugin.name, () => { const hooks: PhasedCommandHooks = new PhasedCommandHooks(); new BuildPlanPlugin(terminal).apply(hooks); - const inputSnapshot: Pick = { + const inputsSnapshot: Pick = { getTrackedFileHashesForOperation() { return new Map(); } }; - const context: Pick = { - inputSnapshot: inputSnapshot as unknown as IInputSnapshot, + const context: Pick = { + inputsSnapshot: inputsSnapshot as unknown as IInputsSnapshot, projectConfigurations: new Map() }; const buildCommand: IPhasedCommandConfig = commandLineConfiguration.commands.get( diff --git a/libraries/rush-lib/src/logic/snapshots/test/__snapshots__/InputSnapshot.test.ts.snap b/libraries/rush-lib/src/logic/snapshots/test/__snapshots__/InputSnapshot.test.ts.snap deleted file mode 100644 index 3960d128ff8..00000000000 --- a/libraries/rush-lib/src/logic/snapshots/test/__snapshots__/InputSnapshot.test.ts.snap +++ /dev/null @@ -1,52 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`InputSnapshot getOperationOwnStateHash Detects outputFileNames collisions 1`] = `"Configured output folder \\"lib\\" for operation \\"_phase:build\\" in project \\"a\\" contains tracked input file \\"a/lib/file3.js\\". If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from \\"outputFolderNames\\". This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder."`; - -exports[`InputSnapshot getOperationOwnStateHash Handles trivial input 1`] = `"acf14e45ed255b0288e449432d48c285f619d146"`; - -exports[`InputSnapshot getOperationOwnStateHash Respects additionalOutputFilesByOperationName 1`] = `"07f4147294d21a07865de84875d60bfbcc690652"`; - -exports[`InputSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 1`] = `"ad1f915d0ac6331c09febbbe1496c970a5401b73"`; - -exports[`InputSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 2`] = `"2c68d56fc9278b6495496070a6a992b929c37a83"`; - -exports[`InputSnapshot getOperationOwnStateHash Respects globalAdditionalFiles 1`] = `"0e0437ad1941bacd098b22da15dc673f86ca6003"`; - -exports[`InputSnapshot getOperationOwnStateHash Respects incrementalBuildIgnoredGlobs 1`] = `"f7b5af9ffdaa39831ed3374f28d0f7dccbee9c8d"`; - -exports[`InputSnapshot getOperationOwnStateHash Respects incrementalBuildIgnoredGlobs 2`] = `"24047d4271ebf8badc9403c9a21a09fb6db2fb9c"`; - -exports[`InputSnapshot getTrackedFileHashesForOperation Detects outputFileNames collisions 1`] = `"Configured output folder \\"lib\\" for operation \\"_phase:build\\" in project \\"a\\" contains tracked input file \\"a/lib/file3.js\\". If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from \\"outputFolderNames\\". This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder."`; - -exports[`InputSnapshot getTrackedFileHashesForOperation Handles trivial input 1`] = ` -Map { - "a/file1.js" => "hash1", - "a/file2.js" => "hash2", - "a/lib/file3.js" => "hash3", -} -`; - -exports[`InputSnapshot getTrackedFileHashesForOperation Respects additionalFilesByOperationName 1`] = ` -Map { - "/ext/config.json" => "hash4", - "a/file1.js" => "hash1", - "a/file2.js" => "hash2", - "a/lib/file3.js" => "hash3", -} -`; - -exports[`InputSnapshot getTrackedFileHashesForOperation Respects globalAdditionalFiles 1`] = ` -Map { - "a/file1.js" => "hash1", - "a/file2.js" => "hash2", - "a/lib/file3.js" => "hash3", - "common/config/some-config.json" => "hash5", -} -`; - -exports[`InputSnapshot getTrackedFileHashesForOperation Respects incrementalBuildIgnoredGlobs 1`] = ` -Map { - "a/file1.js" => "hash1", - "a/lib/file3.js" => "hash3", -} -`; diff --git a/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts b/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts index 63aab041323..fc9e983494e 100644 --- a/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts +++ b/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts @@ -43,9 +43,9 @@ jest.mock(`@rushstack/package-deps-hash`, () => { const mockSnapshot: jest.Mock = jest.fn(); -jest.mock('../snapshots/InputSnapshot', () => { +jest.mock('../incremental/InputsSnapshot', () => { return { - InputSnapshot: mockSnapshot + InputsSnapshot: mockSnapshot }; }); @@ -56,10 +56,10 @@ import { StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import { RushConfiguration } from '../../api/RushConfiguration'; import type { - IInputSnapshot, - IInputSnapshotProvider, - IRushSnapshotParameters -} from '../snapshots/InputSnapshot'; + IInputsSnapshot, + GetInputsSnapshotAsyncFn, + IInputsSnapshotParameters +} from '../incremental/InputsSnapshot'; describe(ProjectChangeAnalyzer.name, () => { beforeEach(() => { @@ -77,9 +77,9 @@ describe(ProjectChangeAnalyzer.name, () => { const terminal: Terminal = new Terminal(terminalProvider); const mockSnapshotValue: {} = {}; mockSnapshot.mockImplementation(() => mockSnapshotValue); - const snapshotProvider: IInputSnapshotProvider | undefined = + const snapshotProvider: GetInputsSnapshotAsyncFn | undefined = await projectChangeAnalyzer._tryGetSnapshotProviderAsync(new Map(), terminal); - const snapshot: IInputSnapshot | undefined = await snapshotProvider?.(); + const snapshot: IInputsSnapshot | undefined = await snapshotProvider?.(); expect(snapshot).toBe(mockSnapshotValue); expect(terminalProvider.getErrorOutput()).toEqual(''); @@ -87,7 +87,7 @@ describe(ProjectChangeAnalyzer.name, () => { expect(mockSnapshot).toHaveBeenCalledTimes(1); - const mockInput: IRushSnapshotParameters = mockSnapshot.mock.calls[0][0]; + const mockInput: IInputsSnapshotParameters = mockSnapshot.mock.calls[0][0]; expect(mockInput.globalAdditionalFiles).toBeDefined(); expect(mockInput.globalAdditionalFiles).toMatchObject(['common/config/rush/npm-shrinkwrap.json']); diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 01dec30d655..c0e9e7f764c 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -24,7 +24,7 @@ import type { RushProjectConfiguration } from '../api/RushProjectConfiguration'; import type { IOperationRunnerContext } from '../logic/operations/IOperationRunner'; import type { ITelemetryData } from '../logic/Telemetry'; import type { OperationStatus } from '../logic/operations/OperationStatus'; -import type { IInputSnapshot } from '../logic/snapshots/InputSnapshot'; +import type { IInputsSnapshot } from '../logic/incremental/InputsSnapshot'; /** * A plugin that interacts with a phased commands. @@ -111,7 +111,7 @@ export interface IExecuteOperationsContext extends ICreateOperationsContext { * The current state of the repository, if available. * Not part of the creation context to avoid the overhead of Git calls when initializing the graph. */ - readonly inputSnapshot?: IInputSnapshot; + readonly inputsSnapshot?: IInputsSnapshot; } /** From 03e19ccf811c0a4c2f478597a90f93e77c0f077b Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 30 Sep 2024 21:33:02 +0000 Subject: [PATCH 3/6] [rush] More renames, move logging --- .../cli/scriptActions/PhasedScriptAction.ts | 23 +++++------ .../src/logic/ProjectChangeAnalyzer.ts | 4 ++ .../rush-lib/src/logic/ProjectWatcher.ts | 38 +++++++++---------- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index e5c0d6a5a40..4436ddfcb7f 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -81,7 +81,7 @@ interface IInitialRunPhasesOptions { } interface IRunPhasesOptions extends IInitialRunPhasesOptions { - snapshotProvider: GetInputsSnapshotAsyncFn | undefined; + getInputsSnapshotAsync: GetInputsSnapshotAsyncFn | undefined; initialSnapshot: IInputsSnapshot | undefined; executionManagerOptions: IOperationExecutionManagerOptions; } @@ -550,17 +550,12 @@ export class PhasedScriptAction extends BaseScriptAction { repoStateStopwatch.start(); const analyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); - const snapshotProvider: GetInputsSnapshotAsyncFn | undefined = + const getInputsSnapshotAsync: GetInputsSnapshotAsyncFn | undefined = await analyzer._tryGetSnapshotProviderAsync(projectConfigurations, terminal); - const initialSnapshot: IInputsSnapshot | undefined = await snapshotProvider?.(); + const initialSnapshot: IInputsSnapshot | undefined = await getInputsSnapshotAsync?.(); repoStateStopwatch.stop(); terminal.writeLine(`DONE (${repoStateStopwatch.toString()})`); - if (!initialSnapshot) { - terminal.writeLine( - `The Rush monorepo is not in a Git repository. Rush will proceed without incremental build support.` - ); - } terminal.writeLine(); const initialExecuteOperationsContext: IExecuteOperationsContext = { @@ -589,7 +584,7 @@ export class PhasedScriptAction extends BaseScriptAction { return { ...options, executionManagerOptions, - snapshotProvider, + getInputsSnapshotAsync, initialSnapshot }; } @@ -666,8 +661,8 @@ export class PhasedScriptAction extends BaseScriptAction { */ private async _runWatchPhasesAsync(options: IRunPhasesOptions): Promise { const { - snapshotProvider, - initialSnapshot: initialState, + getInputsSnapshotAsync, + initialSnapshot, initialCreateOperationsContext, executionManagerOptions, stopwatch, @@ -679,7 +674,7 @@ export class PhasedScriptAction extends BaseScriptAction { const { projectSelection: projectsToWatch } = initialCreateOperationsContext; - if (!snapshotProvider || !initialState) { + if (!getInputsSnapshotAsync || !initialSnapshot) { terminal.writeErrorLine( `Cannot watch for changes if the Rush repo is not in a Git repository, exiting.` ); @@ -693,8 +688,8 @@ export class PhasedScriptAction extends BaseScriptAction { ); const projectWatcher: typeof ProjectWatcher.prototype = new ProjectWatcher({ - getInputSnapshotAsync: snapshotProvider, - initialState, + getInputsSnapshotAsync, + initialSnapshot, debounceMs: this._watchDebounceMs, rushConfiguration: this.rushConfiguration, projectsToWatch, diff --git a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts index 17e699e21d3..942cbac6bdd 100644 --- a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts +++ b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts @@ -207,6 +207,10 @@ export class ProjectChangeAnalyzer { const gitPath: string = this._git.getGitPathOrThrow(); if (!this._git.isPathUnderGitWorkingTree()) { + terminal.writeLine( + `The Rush monorepo is not in a Git repository. Rush will proceed without incremental build support.` + ); + return; } diff --git a/libraries/rush-lib/src/logic/ProjectWatcher.ts b/libraries/rush-lib/src/logic/ProjectWatcher.ts index a123726b1cd..83dd92b904c 100644 --- a/libraries/rush-lib/src/logic/ProjectWatcher.ts +++ b/libraries/rush-lib/src/logic/ProjectWatcher.ts @@ -15,12 +15,12 @@ import type { RushConfiguration } from '../api/RushConfiguration'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; export interface IProjectWatcherOptions { - getInputSnapshotAsync: GetInputsSnapshotAsyncFn; + getInputsSnapshotAsync: GetInputsSnapshotAsyncFn; debounceMs?: number; rushConfiguration: RushConfiguration; projectsToWatch: ReadonlySet; terminal: ITerminal; - initialState?: IInputsSnapshot | undefined; + initialSnapshot?: IInputsSnapshot | undefined; } export interface IProjectChangeResult { @@ -60,8 +60,8 @@ export class ProjectWatcher { private readonly _projectsToWatch: ReadonlySet; private readonly _terminal: ITerminal; - private _initialState: IInputsSnapshot | undefined; - private _previousState: IInputsSnapshot | undefined; + private _initialSnapshot: IInputsSnapshot | undefined; + private _previousSnapshot: IInputsSnapshot | undefined; private _forceChangedProjects: Map = new Map(); private _resolveIfChanged: undefined | (() => Promise); private _getPromptLines: undefined | IPromptGeneratorFunction; @@ -72,12 +72,12 @@ export class ProjectWatcher { public constructor(options: IProjectWatcherOptions) { const { - getInputSnapshotAsync: snapshotProvider, + getInputsSnapshotAsync: snapshotProvider, debounceMs = 1000, rushConfiguration, projectsToWatch, terminal, - initialState + initialSnapshot: initialState } = options; this._debounceMs = debounceMs; @@ -88,8 +88,8 @@ export class ProjectWatcher { const gitPath: string = new Git(rushConfiguration).getGitPathOrThrow(); this._repoRoot = Path.convertToSlashes(getRepoRoot(rushConfiguration.rushJsonFolder, gitPath)); - this._initialState = initialState; - this._previousState = initialState; + this._initialSnapshot = initialState; + this._previousSnapshot = initialState; this._renderedStatusLines = 0; this._getPromptLines = undefined; @@ -400,27 +400,27 @@ export class ProjectWatcher { * Determines which, if any, projects (within the selection) have new hashes for files that are not in .gitignore */ private async _computeChangedAsync(): Promise { - const state: IInputsSnapshot | undefined = await this._getInputsSnapshotAsync(); + const currentSnapshot: IInputsSnapshot | undefined = await this._getInputsSnapshotAsync(); - if (!state) { + if (!currentSnapshot) { throw new AlreadyReportedError(); } - const previousState: IInputsSnapshot | undefined = this._previousState; + const previousSnapshot: IInputsSnapshot | undefined = this._previousSnapshot; - if (!previousState) { + if (!previousSnapshot) { return { changedProjects: this._projectsToWatch, - inputsSnapshot: state + inputsSnapshot: currentSnapshot }; } const changedProjects: Set = new Set(); for (const project of this._projectsToWatch) { const previous: ReadonlyMap | undefined = - previousState.getTrackedFileHashesForOperation(project); + previousSnapshot.getTrackedFileHashesForOperation(project); const current: ReadonlyMap | undefined = - state.getTrackedFileHashesForOperation(project); + currentSnapshot.getTrackedFileHashesForOperation(project); if (ProjectWatcher._haveProjectDepsChanged(previous, current)) { // May need to detect if the nature of the change will break the process, e.g. changes to package.json @@ -434,14 +434,14 @@ export class ProjectWatcher { return { changedProjects, - inputsSnapshot: state + inputsSnapshot: currentSnapshot }; } private _commitChanges(state: IInputsSnapshot): void { - this._previousState = state; - if (!this._initialState) { - this._initialState = state; + this._previousSnapshot = state; + if (!this._initialSnapshot) { + this._initialSnapshot = state; } } From f4405204f174c91919ea4e3cd4968de3f9d61ee5 Mon Sep 17 00:00:00 2001 From: David Michon Date: Thu, 3 Oct 2024 23:23:34 +0000 Subject: [PATCH 4/6] [rush] Allow rush plugins to add metadata files to cache entries --- .../config/heft.json | 10 +++++ .../src/test-metadata.ts | 14 ++++++ .../metadata-folder_2024-10-03-23-13.json | 10 +++++ common/reviews/api/rush-lib.api.md | 3 +- .../src/logic/buildCache/ProjectBuildCache.ts | 29 +++--------- .../operations/CacheableOperationPlugin.ts | 45 ++++++++----------- .../operations/IOperationExecutionResult.ts | 4 ++ .../operations/OperationExecutionRecord.ts | 7 ++- .../operations/OperationMetadataManager.ts | 21 +++------ 9 files changed, 77 insertions(+), 66 deletions(-) create mode 100644 build-tests/heft-node-everything-test/src/test-metadata.ts create mode 100644 common/changes/@microsoft/rush/metadata-folder_2024-10-03-23-13.json diff --git a/build-tests/heft-node-everything-test/config/heft.json b/build-tests/heft-node-everything-test/config/heft.json index 41b92441f9d..fc24874e5d3 100644 --- a/build-tests/heft-node-everything-test/config/heft.json +++ b/build-tests/heft-node-everything-test/config/heft.json @@ -26,6 +26,16 @@ "taskPlugin": { "pluginPackage": "@rushstack/heft-api-extractor-plugin" } + }, + "metadata-test": { + "taskDependencies": ["typescript"], + "taskPlugin": { + "pluginPackage": "@rushstack/heft", + "pluginName": "run-script-plugin", + "options": { + "scriptPath": "./lib/test-metadata.js" + } + } } } }, diff --git a/build-tests/heft-node-everything-test/src/test-metadata.ts b/build-tests/heft-node-everything-test/src/test-metadata.ts new file mode 100644 index 00000000000..1fd618c599a --- /dev/null +++ b/build-tests/heft-node-everything-test/src/test-metadata.ts @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs/promises'; + +import type { IRunScriptOptions } from '@rushstack/heft'; + +export async function runAsync({ heftConfiguration: { buildFolderPath } }: IRunScriptOptions): Promise { + const metadataFolder: string = `${buildFolderPath}/.rush/temp/operation/_phase_build`; + + await fs.mkdir(metadataFolder, { recursive: true }); + + await fs.writeFile(`${metadataFolder}/test.txt`, new Date().toString(), 'utf-8'); +} diff --git a/common/changes/@microsoft/rush/metadata-folder_2024-10-03-23-13.json b/common/changes/@microsoft/rush/metadata-folder_2024-10-03-23-13.json new file mode 100644 index 00000000000..94a9913dda7 --- /dev/null +++ b/common/changes/@microsoft/rush/metadata-folder_2024-10-03-23-13.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Allow rush plugins to extend build cache entries by writing additional files to the metadata folder. Expose the metadata folder path to plugins.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 81df579ecf4..81561784cc8 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -581,6 +581,7 @@ export interface IOperationExecutionResult { readonly cobuildRunnerId: string | undefined; readonly error: Error | undefined; readonly logFilePaths: ILogFilePaths | undefined; + readonly metadataFolderPath: string | undefined; readonly nonCachedDurationMs: number | undefined; readonly operation: Operation; readonly status: OperationStatus; @@ -941,7 +942,7 @@ export class _OperationMetadataManager { constructor(options: _IOperationMetadataManagerOptions); // (undocumented) readonly logFilenameIdentifier: string; - get relativeFilepaths(): string[]; + get metadataFolderPath(): string; // (undocumented) saveAsync({ durationInSeconds, cobuildContextId, cobuildRunnerId, logPath, errorLogPath, logChunksPath }: _IOperationMetadata): Promise; // (undocumented) diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index f4efa538612..cbba6950d83 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -27,10 +27,6 @@ export interface IProjectBuildCacheOptions { * Value from rush-project.json */ projectOutputFolderNames: ReadonlyArray; - /** - * Value from CacheableOperationPlugin - */ - additionalProjectOutputFilePaths?: ReadonlyArray; /** * The hash of all relevant inputs and configuration that uniquely identifies this execution. */ @@ -59,7 +55,6 @@ export class ProjectBuildCache { private readonly _buildCacheEnabled: boolean; private readonly _cacheWriteEnabled: boolean; private readonly _projectOutputFolderNames: ReadonlyArray; - private readonly _additionalProjectOutputFilePaths: ReadonlyArray; private readonly _cacheId: string | undefined; private constructor(cacheId: string | undefined, options: IProjectBuildCacheOptions) { @@ -71,8 +66,7 @@ export class ProjectBuildCache { cacheWriteEnabled }, project, - projectOutputFolderNames, - additionalProjectOutputFilePaths + projectOutputFolderNames } = options; this._project = project; this._localBuildCacheProvider = localCacheProvider; @@ -80,7 +74,6 @@ export class ProjectBuildCache { this._buildCacheEnabled = buildCacheEnabled; this._cacheWriteEnabled = cacheWriteEnabled; this._projectOutputFolderNames = projectOutputFolderNames || []; - this._additionalProjectOutputFilePaths = additionalProjectOutputFilePaths || []; this._cacheId = cacheId; } @@ -360,19 +353,6 @@ export class ProjectBuildCache { return undefined; } - // Add additional output file paths - await Async.forEachAsync( - this._additionalProjectOutputFilePaths, - async (additionalProjectOutputFilePath: string) => { - const fullPath: string = `${projectFolderPath}/${additionalProjectOutputFilePath}`; - const pathExists: boolean = await FileSystem.existsAsync(fullPath); - if (pathExists) { - outputFilePaths.push(additionalProjectOutputFilePath); - } - }, - { concurrency: 10 } - ); - // Ensure stable output path order. outputFilePaths.sort(); @@ -387,7 +367,12 @@ export class ProjectBuildCache { } private static _getCacheId(options: IProjectBuildCacheOptions): string | undefined { - const { buildCacheConfiguration, project: { packageName }, operationStateHash, phaseName } = options; + const { + buildCacheConfiguration, + project: { packageName }, + operationStateHash, + phaseName + } = options; return buildCacheConfiguration.getCacheEntryId({ projectName: packageName, projectStateHash: operationStateHash, diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index e680a7e0446..c43672698d7 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -32,7 +32,6 @@ import type { PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; import type { IPhase } from '../../api/CommandLineConfiguration'; -import type { OperationMetadataManager } from './OperationMetadataManager'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; import type { OperationExecutionRecord } from './OperationExecutionRecord'; @@ -54,7 +53,7 @@ export interface IOperationBuildCacheContext { operationBuildCache: ProjectBuildCache | undefined; cacheDisabledReason: string | undefined; - outputFolderNames: ReadonlyArray | undefined; + outputFolderNames: ReadonlyArray; cobuildLock: CobuildLock | undefined; @@ -166,8 +165,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return `${RushConstants.hashDelimiter}${operation.name}=${getOrCreateOperationHash(operation)}`; } - for (const operation of recordByOperation.keys()) { - const { associatedProject, associatedPhase, runner } = operation; + for (const [operation, record] of recordByOperation) { + const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation; if (!associatedProject || !associatedPhase || !runner) { return; } @@ -188,8 +187,15 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { : `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + 'or one provided by a rig, so it does not support caching.'; - const outputFolderNames: ReadonlyArray | undefined = - projectConfiguration?.operationSettingsByOperationName.get(phaseName)?.outputFolderNames; + const metadataFolderPath: string | undefined = record.metadataFolderPath; + + const outputFolderNames: string[] = metadataFolderPath ? [metadataFolderPath] : []; + const configuredOutputFolderNames: string[] | undefined = operationSettings?.outputFolderNames; + if (configuredOutputFolderNames) { + for (const folderName of configuredOutputFolderNames) { + outputFolderNames.push(folderName); + } + } disjointSet?.add(operation); @@ -273,7 +279,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { associatedProject: project, associatedPhase: phase, runner, - _operationMetadataManager: operationMetadataManager + _operationMetadataManager: operationMetadataManager, + operation } = record; if ( @@ -312,8 +319,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { rushProject: project, phase, terminal: buildCacheTerminal, - operationMetadataManager, - operation: record.operation + operation: operation }); // Try to acquire the cobuild lock @@ -321,7 +327,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (cobuildConfiguration?.cobuildFeatureEnabled) { if ( cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && - record.operation.consumers.size === 0 && + operation.consumers.size === 0 && !projectBuildCache ) { // When the leaf project log only is allowed and the leaf project is build cache "disabled", try to get @@ -332,8 +338,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheContext, rushProject: project, phase, - terminal: buildCacheTerminal, - operationMetadataManager + terminal: buildCacheTerminal }); if (projectBuildCache) { buildCacheTerminal.writeVerboseLine( @@ -628,7 +633,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { rushProject, phase, terminal, - operationMetadataManager, operation }: { buildCacheContext: IOperationBuildCacheContext; @@ -636,7 +640,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { rushProject: RushConfigurationProject; phase: IPhase; terminal: ITerminal; - operationMetadataManager: OperationMetadataManager | undefined; operation: Operation; }): ProjectBuildCache | undefined { if (!buildCacheContext.operationBuildCache) { @@ -652,14 +655,10 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return; } - const additionalProjectOutputFilePaths: ReadonlyArray = - operationMetadataManager?.relativeFilepaths || []; - // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent buildCacheContext.operationBuildCache = ProjectBuildCache.getProjectBuildCache({ project: rushProject, projectOutputFolderNames: outputFolderNames, - additionalProjectOutputFilePaths, buildCacheConfiguration, terminal, operationStateHash, @@ -677,8 +676,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { terminal, buildCacheConfiguration, cobuildConfiguration, - phase, - operationMetadataManager + phase }: { buildCacheContext: IOperationBuildCacheContext; buildCacheConfiguration: BuildCacheConfiguration | undefined; @@ -686,7 +684,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { rushProject: RushConfigurationProject; phase: IPhase; terminal: ITerminal; - operationMetadataManager: OperationMetadataManager | undefined; }): Promise { if (!buildCacheConfiguration?.buildCacheEnabled) { return; @@ -694,9 +691,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const { outputFolderNames, stateHash } = buildCacheContext; - const additionalProjectOutputFilePaths: ReadonlyArray = - operationMetadataManager?.relativeFilepaths || []; - const hasher: crypto.Hash = crypto.createHash('sha1'); hasher.update(stateHash); @@ -710,8 +704,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const projectBuildCache: ProjectBuildCache = ProjectBuildCache.getProjectBuildCache({ project: rushProject, - projectOutputFolderNames: outputFolderNames || [], - additionalProjectOutputFilePaths, + projectOutputFolderNames: outputFolderNames, buildCacheConfiguration, terminal, operationStateHash, diff --git a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts index 578c3c0fe8b..9aa5efdcf88 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts @@ -44,6 +44,10 @@ export interface IOperationExecutionResult { * The id of the runner which actually runs the building process in cobuild mode. */ readonly cobuildRunnerId: string | undefined; + /** + * The relative path to the folder that contains operation metadata. This folder will be automatically included in cache entries. + */ + readonly metadataFolderPath: string | undefined; /** * The paths to the log files, if applicable. */ diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index efc93a3e39d..aa8956f14da 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -28,6 +28,7 @@ import { type ILogFilePaths, initializeProjectLogFilesAsync } from './ProjectLogWritable'; +import type { IOperationExecutionResult } from './IOperationExecutionResult'; export interface IOperationExecutionRecordContext { streamCollator: StreamCollator; @@ -42,7 +43,7 @@ export interface IOperationExecutionRecordContext { * * @internal */ -export class OperationExecutionRecord implements IOperationRunnerContext { +export class OperationExecutionRecord implements IOperationRunnerContext, IOperationExecutionResult { /** * The associated operation. */ @@ -176,6 +177,10 @@ export class OperationExecutionRecord implements IOperationRunnerContext { return this._operationMetadataManager?.stateFile.state?.cobuildRunnerId; } + public get metadataFolderPath(): string | undefined { + return this._operationMetadataManager?.metadataFolderPath; + } + public get isTerminal(): boolean { return TERMINAL_STATUSES.has(this.status); } diff --git a/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts b/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts index 6c722357850..a327ae977dc 100644 --- a/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts @@ -56,9 +56,6 @@ export class OperationMetadataManager { private readonly _logPath: string; private readonly _errorLogPath: string; private readonly _logChunksPath: string; - private readonly _relativeLogPath: string; - private readonly _relativeLogChunksPath: string; - private readonly _relativeErrorLogPath: string; public constructor(options: IOperationMetadataManagerOptions) { const { @@ -77,12 +74,9 @@ export class OperationMetadataManager { }); this._metadataFolderPath = metadataFolderPath; - this._relativeLogPath = `${metadataFolderPath}/all.log`; - this._relativeErrorLogPath = `${metadataFolderPath}/error.log`; - this._relativeLogChunksPath = `${metadataFolderPath}/log-chunks.jsonl`; - this._logPath = `${projectFolder}/${this._relativeLogPath}`; - this._errorLogPath = `${projectFolder}/${this._relativeErrorLogPath}`; - this._logChunksPath = `${projectFolder}/${this._relativeLogChunksPath}`; + this._logPath = `${projectFolder}/${metadataFolderPath}/all.log`; + this._errorLogPath = `${projectFolder}/${metadataFolderPath}/error.log`; + this._logChunksPath = `${projectFolder}/${metadataFolderPath}/log-chunks.jsonl`; } /** @@ -92,13 +86,8 @@ export class OperationMetadataManager { * Example: `.rush/temp/operation/_phase_build/all.log` * Example: `.rush/temp/operation/_phase_build/error.log` */ - public get relativeFilepaths(): string[] { - return [ - this.stateFile.relativeFilepath, - this._relativeLogPath, - this._relativeErrorLogPath, - this._relativeLogChunksPath - ]; + public get metadataFolderPath(): string { + return this._metadataFolderPath; } public async saveAsync({ From 4edbbb8abd6c0cf2b98339555a5d321280043799 Mon Sep 17 00:00:00 2001 From: David Michon Date: Fri, 4 Oct 2024 00:56:08 +0000 Subject: [PATCH 5/6] [rush] Add `operation.enabled` flag --- .../heft-node-everything-test/package.json | 4 +- ...ject-change-analyzer_2024-10-04-00-55.json | 10 + common/reviews/api/rush-lib.api.md | 2 + .../cli/scriptActions/PhasedScriptAction.ts | 27 +- .../operations/CacheableOperationPlugin.ts | 11 +- .../logic/operations/ConsoleTimelinePlugin.ts | 2 +- .../operations/IOperationExecutionResult.ts | 4 + .../src/logic/operations/Operation.ts | 10 + .../operations/OperationExecutionManager.ts | 16 +- .../operations/OperationExecutionRecord.ts | 7 +- .../OperationResultSummarizerPlugin.ts | 2 +- .../logic/operations/PhasedOperationPlugin.ts | 18 +- .../test/PhasedOperationPlugin.test.ts | 2 +- .../PhasedOperationPlugin.test.ts.snap | 808 +++++++++--------- .../rush-serve-plugin/src/api.types.ts | 7 + .../src/phasedCommandHandler.ts | 5 +- 16 files changed, 488 insertions(+), 447 deletions(-) create mode 100644 common/changes/@microsoft/rush/split-project-change-analyzer_2024-10-04-00-55.json diff --git a/build-tests/heft-node-everything-test/package.json b/build-tests/heft-node-everything-test/package.json index 5163114f745..a8bf85ffb45 100644 --- a/build-tests/heft-node-everything-test/package.json +++ b/build-tests/heft-node-everything-test/package.json @@ -8,7 +8,9 @@ "scripts": { "build": "heft build --clean", "_phase:build": "heft run --only build -- --clean", - "_phase:test": "heft run --only test -- --clean" + "_phase:build:incremental": "heft run --only build --", + "_phase:test": "heft run --only test -- --clean", + "_phase:test:incremental": "heft run --only test --" }, "devDependencies": { "@microsoft/api-extractor": "workspace:*", diff --git a/common/changes/@microsoft/rush/split-project-change-analyzer_2024-10-04-00-55.json b/common/changes/@microsoft/rush/split-project-change-analyzer_2024-10-04-00-55.json new file mode 100644 index 00000000000..5c43e3ba71c --- /dev/null +++ b/common/changes/@microsoft/rush/split-project-change-analyzer_2024-10-04-00-55.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Add a new property `enabled` to `Operation` that when set to false, will cause the execution engine to immediately return `OperationStatus.Skipped` instead of invoking the runner. Use this property to disable operations that are not intended to be executed in the current pass, e.g. those that did not contain changes in the most recent watch iteration, or those excluded by `--only`.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 81561784cc8..8ea631c00c6 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -584,6 +584,7 @@ export interface IOperationExecutionResult { readonly metadataFolderPath: string | undefined; readonly nonCachedDurationMs: number | undefined; readonly operation: Operation; + readonly silent: boolean; readonly status: OperationStatus; readonly stdioSummarizer: StdioSummarizer; readonly stopwatch: IStopwatchResult; @@ -929,6 +930,7 @@ export class Operation { readonly consumers: ReadonlySet; deleteDependency(dependency: Operation): void; readonly dependencies: ReadonlySet; + enabled: boolean; get isNoOp(): boolean; logFilenameIdentifier: string; get name(): string | undefined; diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 4436ddfcb7f..c70ded04928 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -38,7 +38,10 @@ import { ShellOperationRunnerPlugin } from '../../logic/operations/ShellOperatio import { Event } from '../../api/EventHooks'; import { ProjectChangeAnalyzer } from '../../logic/ProjectChangeAnalyzer'; import { OperationStatus } from '../../logic/operations/OperationStatus'; -import type { IExecutionResult } from '../../logic/operations/IOperationExecutionResult'; +import type { + IExecutionResult, + IOperationExecutionResult +} from '../../logic/operations/IOperationExecutionResult'; import { OperationResultSummarizerPlugin } from '../../logic/operations/OperationResultSummarizerPlugin'; import type { ITelemetryData, ITelemetryOperationResult } from '../../logic/Telemetry'; import { parseParallelism } from '../parsing/ParseParallelism'; @@ -53,6 +56,7 @@ import type { ProjectWatcher } from '../../logic/ProjectWatcher'; import { FlagFile } from '../../api/FlagFile'; import { WeightedOperationPlugin } from '../../logic/operations/WeightedOperationPlugin'; import { getVariantAsync, VARIANT_PARAMETER } from '../../api/Variants'; +import { Selection } from '../../logic/Selection'; /** * Constructor parameters for PhasedScriptAction. @@ -472,10 +476,13 @@ export class PhasedScriptAction extends BaseScriptAction { new PnpmSyncCopyOperationPlugin(terminal).apply(this.hooks); } + const relevantProjects: Set = + Selection.expandAllDependencies(projectSelection); + const projectConfigurations: ReadonlyMap = this ._runsBeforeInstall ? new Map() - : await RushProjectConfiguration.tryLoadForProjectsAsync(projectSelection, terminal); + : await RushProjectConfiguration.tryLoadForProjectsAsync(relevantProjects, terminal); const initialCreateOperationsContext: ICreateOperationsContext = { buildCacheConfiguration, @@ -840,7 +847,7 @@ export class PhasedScriptAction extends BaseScriptAction { } if (this.parser.telemetry) { - const operationResults: Record = {}; + const jsonOperationResults: Record = {}; const extraData: IPhasedCommandTelemetry = { // Fields preserved across the command invocation @@ -861,6 +868,8 @@ export class PhasedScriptAction extends BaseScriptAction { }; if (result) { + const { operationResults } = result; + const nonSilentDependenciesByOperation: Map> = new Map(); function getNonSilentDependencies(operation: Operation): ReadonlySet { let realDependencies: Set | undefined = nonSilentDependenciesByOperation.get(operation); @@ -868,7 +877,9 @@ export class PhasedScriptAction extends BaseScriptAction { realDependencies = new Set(); nonSilentDependenciesByOperation.set(operation, realDependencies); for (const dependency of operation.dependencies) { - if (dependency.runner!.silent) { + const dependencyRecord: IOperationExecutionResult | undefined = + operationResults.get(dependency); + if (dependencyRecord?.silent) { for (const deepDependency of getNonSilentDependencies(dependency)) { realDependencies.add(deepDependency); } @@ -880,14 +891,14 @@ export class PhasedScriptAction extends BaseScriptAction { return realDependencies; } - for (const [operation, operationResult] of result.operationResults) { - if (operation.runner?.silent) { + for (const [operation, operationResult] of operationResults) { + if (operationResult.silent) { // Architectural operation. Ignore. continue; } const { startTime, endTime } = operationResult.stopwatch; - operationResults[operation.name!] = { + jsonOperationResults[operation.name!] = { startTimestampMs: startTime, endTimestampMs: endTime, nonCachedDurationMs: operationResult.nonCachedDurationMs, @@ -930,7 +941,7 @@ export class PhasedScriptAction extends BaseScriptAction { durationInSeconds: stopwatch.duration, result: success ? 'Succeeded' : 'Failed', extraData, - operationResults + operationResults: jsonOperationResults }; this.hooks.beforeLog.call(logEntry); diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index c43672698d7..270d82dc2c3 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -284,6 +284,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } = record; if ( + !operation.enabled || !project || !phase || !runner?.cacheable || @@ -305,7 +306,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheContext, buildCacheEnabled: buildCacheConfiguration?.buildCacheEnabled, rushProject: project, - logFilenameIdentifier: operationMetadataManager.logFilenameIdentifier, + logFilenameIdentifier: operation.logFilenameIdentifier, quietMode: record.quietMode, debugMode: record.debugMode }); @@ -378,7 +379,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const { error: errorLogPath } = getProjectLogFilePaths({ project, - logFilenameIdentifier: operationMetadataManager.logFilenameIdentifier + logFilenameIdentifier: operation.logFilenameIdentifier }); const restoreCacheAsync = async ( // TODO: Investigate if `projectBuildCacheForRestore` is always the same instance as `projectBuildCache` @@ -473,9 +474,9 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; const { status, stopwatch, _operationMetadataManager: operationMetadataManager, operation } = record; - const { associatedProject: project, associatedPhase: phase, runner } = operation; + const { associatedProject: project, associatedPhase: phase, runner, enabled } = operation; - if (!project || !phase || !runner?.cacheable || !operationMetadataManager) { + if (!enabled || !project || !phase || !runner?.cacheable || !operationMetadataManager) { return; } @@ -766,7 +767,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { quietMode: boolean; debugMode: boolean; }): Promise { - const silent: boolean = record.runner.silent; + const silent: boolean = record.silent; if (silent) { const nullTerminalProvider: NullTerminalProvider = new NullTerminalProvider(); return new Terminal(nullTerminalProvider); diff --git a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts index 8ad9f7d0663..5931ce1694b 100644 --- a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts @@ -150,7 +150,7 @@ export function _printTimeline({ terminal, result, cobuildConfiguration }: IPrin let workDuration: number = 0; for (const [operation, operationResult] of result.operationResults) { - if (operation.runner?.silent) { + if (operationResult.silent) { continue; } diff --git a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts index 9aa5efdcf88..6991beb5ccb 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts @@ -28,6 +28,10 @@ export interface IOperationExecutionResult { * it later (for example to re-print errors at end of execution). */ readonly error: Error | undefined; + /** + * If this operation is only present in the graph to maintain dependency relationships, this flag will be set to true. + */ + readonly silent: boolean; /** * Object tracking execution timing. */ diff --git a/libraries/rush-lib/src/logic/operations/Operation.ts b/libraries/rush-lib/src/logic/operations/Operation.ts index 34568f2c06c..bbc8d5d2af8 100644 --- a/libraries/rush-lib/src/logic/operations/Operation.ts +++ b/libraries/rush-lib/src/logic/operations/Operation.ts @@ -15,10 +15,12 @@ export interface IOperationOptions { * The Rush phase associated with this Operation, if any */ phase?: IPhase | undefined; + /** * The Rush project associated with this Operation, if any */ project?: RushConfigurationProject | undefined; + /** * When the scheduler is ready to process this `Operation`, the `runner` implements the actual work of * running the operation. @@ -98,6 +100,13 @@ export class Operation { */ public settings: IOperationSettings | undefined = undefined; + /** + * If set to false, this operation will be skipped during evaluation (return OperationStatus.Skipped). + * This is useful for plugins to alter the scope of the operation graph across executions, + * e.g. to enable or disable unit test execution, or to include or exclude dependencies. + */ + public enabled: boolean; + public constructor(options: IOperationOptions) { const { phase, project, runner, settings, logFilenameIdentifier } = options; this.associatedPhase = phase; @@ -105,6 +114,7 @@ export class Operation { this.runner = runner; this.settings = settings; this.logFilenameIdentifier = logFilenameIdentifier; + this.enabled = true; } /** diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index f1e06dabf49..d564a53eea7 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -138,7 +138,7 @@ export class OperationExecutionManager { ); executionRecords.set(operation, executionRecord); - if (!executionRecord.runner.silent) { + if (!executionRecord.silent) { // Only count non-silent operations totalOperations++; } @@ -212,7 +212,7 @@ export class OperationExecutionManager { this._terminal.writeStdoutLine(`Selected ${totalOperations} operation${plural}:`); const nonSilentOperations: string[] = []; for (const record of this._executionRecords.values()) { - if (!record.runner.silent) { + if (!record.silent) { nonSilentOperations.push(record.name); } } @@ -252,8 +252,8 @@ export class OperationExecutionManager { await Async.forEachAsync( this._executionQueue, - async (operation: OperationExecutionRecord) => { - await operation.executeAsync({ + async (record: OperationExecutionRecord) => { + await record.executeAsync({ onStart: onOperationStartAsync, onResult: onOperationCompleteAsync }); @@ -301,9 +301,7 @@ export class OperationExecutionManager { * Handles the result of the operation and propagates any relevant effects. */ private _onOperationComplete(record: OperationExecutionRecord): void { - const { runner, name, status } = record; - - const silent: boolean = runner.silent; + const { runner, name, status, silent } = record; switch (status) { /** @@ -324,13 +322,13 @@ export class OperationExecutionManager { // Now that we have the concept of architectural no-ops, we could implement this by replacing // {blockedRecord.runner} with a no-op that sets status to Blocked and logs the blocking // operations. However, the existing behavior is a bit simpler, so keeping that for now. - if (!blockedRecord.runner.silent) { + if (!blockedRecord.silent) { terminal.writeStdoutLine(`"${blockedRecord.name}" is blocked by "${name}".`); } blockedRecord.status = OperationStatus.Blocked; this._executionQueue.complete(blockedRecord); - if (!blockedRecord.runner.silent) { + if (!blockedRecord.silent) { // Only increment the count if the operation is not silent to avoid confusing the user. // The displayed total is the count of non-silent operations. this._completedOperations++; diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index aa8956f14da..5aa22ef73be 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -202,6 +202,10 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera this._context.onOperationStatusChanged?.(this); } + public get silent(): boolean { + return !this.operation.enabled || this.runner.silent; + } + /** * {@inheritdoc IOperationRunnerContext.runWithTerminalAsync} */ @@ -308,7 +312,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera if (earlyReturnStatus) { this.status = earlyReturnStatus; } else { - this.status = await this.runner.executeAsync(this); + // If the operation is disabled, skip the runner and directly mark as Skipped. + this.status = this.operation.enabled ? await this.runner.executeAsync(this) : OperationStatus.Skipped; } // Delegate global state reporting await onResult(this); diff --git a/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts b/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts index a0bd482bf19..ec9a3cde168 100644 --- a/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts @@ -52,7 +52,7 @@ export function _printOperationStatus(terminal: ITerminal, result: IExecutionRes const operationsByStatus: IOperationsByStatus = new Map(); for (const record of operationResults) { - if (record[0].runner?.silent) { + if (record[1].silent) { // Don't report silenced operations continue; } diff --git a/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts index d7c1342c910..0eeaaf0f651 100644 --- a/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts @@ -5,8 +5,6 @@ import type { RushConfigurationProject } from '../../api/RushConfigurationProjec import type { IPhase } from '../../api/CommandLineConfiguration'; import { Operation } from './Operation'; -import { OperationStatus } from './OperationStatus'; -import { NullOperationRunner } from './NullOperationRunner'; import type { ICreateOperationsContext, IPhasedCommandPlugin, @@ -55,15 +53,11 @@ function createOperations( } } - for (const [key, operation] of operations) { + for (const operation of operations.values()) { if (!operationsWithWork.has(operation)) { // This operation is in scope, but did not change since it was last executed by the current command. // However, we have no state tracking across executions, so treat as unknown. - operation.runner = new NullOperationRunner({ - name: key, - result: OperationStatus.Skipped, - silent: true - }); + operation.enabled = false; } } @@ -91,12 +85,8 @@ function createOperations( }); if (!phaseSelection.has(phase) || !projectSelection.has(project)) { - // Not in scope. Mark skipped because state is unknown. - operation.runner = new NullOperationRunner({ - name: key, - result: OperationStatus.Skipped, - silent: true - }); + // Not in scope. Mark disabled, which will report as OperationStatus.Skipped. + operation.enabled = false; } else if (changedProjects.has(project)) { operationsWithWork.add(operation); } diff --git a/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts index 89f434a673e..273e86a2313 100644 --- a/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts @@ -30,7 +30,7 @@ interface ISerializedOperation { function serializeOperation(operation: Operation): ISerializedOperation { return { name: operation.name!, - silent: operation.runner!.silent, + silent: !operation.enabled || operation.runner!.silent, dependencies: Array.from(operation.dependencies, (dep: Operation) => dep.name!) }; } diff --git a/libraries/rush-lib/src/logic/operations/test/__snapshots__/PhasedOperationPlugin.test.ts.snap b/libraries/rush-lib/src/logic/operations/test/__snapshots__/PhasedOperationPlugin.test.ts.snap index f0d8741667a..39094171e89 100644 --- a/libraries/rush-lib/src/logic/operations/test/__snapshots__/PhasedOperationPlugin.test.ts.snap +++ b/libraries/rush-lib/src/logic/operations/test/__snapshots__/PhasedOperationPlugin.test.ts.snap @@ -623,19 +623,19 @@ exports[`PhasedOperationPlugin handles filtered phases 1`] = ` Array [ Object { "dependencies": Array [ - "a;_phase:no-deps", + "a (no-deps)", ], "name": "a (upstream-self)", "silent": false, }, Object { "dependencies": Array [], - "name": "a;_phase:no-deps", + "name": "a (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", "a (upstream-self)", ], "name": "b (upstream-self)", @@ -643,12 +643,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "b;_phase:no-deps", + "name": "b (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "c;_phase:no-deps", + "c (no-deps)", "b (upstream-self)", ], "name": "c (upstream-self)", @@ -656,12 +656,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "c;_phase:no-deps", + "name": "c (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "d;_phase:no-deps", + "d (no-deps)", "b (upstream-self)", ], "name": "d (upstream-self)", @@ -669,12 +669,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "d;_phase:no-deps", + "name": "d (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "e;_phase:no-deps", + "e (no-deps)", "c (upstream-self)", ], "name": "e (upstream-self)", @@ -682,12 +682,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "e;_phase:no-deps", + "name": "e (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "f;_phase:no-deps", + "f (no-deps)", "a (upstream-self)", "h (upstream-self)", ], @@ -696,12 +696,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "f;_phase:no-deps", + "name": "f (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "h;_phase:no-deps", + "h (no-deps)", "a (upstream-self)", ], "name": "h (upstream-self)", @@ -709,12 +709,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "h;_phase:no-deps", + "name": "h (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "g;_phase:no-deps", + "g (no-deps)", "a (upstream-self)", ], "name": "g (upstream-self)", @@ -722,31 +722,31 @@ Array [ }, Object { "dependencies": Array [], - "name": "g;_phase:no-deps", + "name": "g (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "i;_phase:no-deps", + "i (no-deps)", ], "name": "i (upstream-self)", "silent": false, }, Object { "dependencies": Array [], - "name": "i;_phase:no-deps", + "name": "i (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "j;_phase:no-deps", + "j (no-deps)", ], "name": "j (upstream-self)", "silent": false, }, Object { "dependencies": Array [], - "name": "j;_phase:no-deps", + "name": "j (no-deps)", "silent": true, }, ] @@ -769,48 +769,48 @@ Array [ Object { "dependencies": Array [ "b (upstream-3)", - "a;_phase:upstream-1-self-upstream", - "a;_phase:upstream-2-self", + "a (upstream-1-self-upstream)", + "a (upstream-2-self)", ], "name": "b (complex)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], "name": "b (upstream-3)", "silent": false, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-2", + "name": "a (upstream-2)", "silent": true, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-1-self-upstream", + "name": "a (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], - "name": "a;_phase:upstream-2-self", + "name": "a (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ "c (upstream-3)", - "b;_phase:upstream-1-self-upstream", - "b;_phase:upstream-2-self", + "b (upstream-1-self-upstream)", + "b (upstream-2-self)", ], "name": "c (complex)", "silent": false, }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], "name": "c (upstream-3)", "silent": false, @@ -819,7 +819,7 @@ Array [ "dependencies": Array [ "a (upstream-1)", ], - "name": "b;_phase:upstream-2", + "name": "b (upstream-2)", "silent": true, }, Object { @@ -829,37 +829,37 @@ Array [ }, Object { "dependencies": Array [ - "a;_phase:upstream-1-self", + "a (upstream-1-self)", ], - "name": "b;_phase:upstream-1-self-upstream", + "name": "b (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ "a (upstream-1)", ], - "name": "a;_phase:upstream-1-self", + "name": "a (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], - "name": "b;_phase:upstream-2-self", + "name": "b (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ "d (upstream-3)", - "b;_phase:upstream-1-self-upstream", - "b;_phase:upstream-2-self", + "b (upstream-1-self-upstream)", + "b (upstream-2-self)", ], "name": "d (complex)", "silent": false, }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], "name": "d (upstream-3)", "silent": false, @@ -867,15 +867,15 @@ Array [ Object { "dependencies": Array [ "e (upstream-3)", - "c;_phase:upstream-1-self-upstream", - "c;_phase:upstream-2-self", + "c (upstream-1-self-upstream)", + "c (upstream-2-self)", ], "name": "e (complex)", "silent": false, }, Object { "dependencies": Array [ - "c;_phase:upstream-2", + "c (upstream-2)", ], "name": "e (upstream-3)", "silent": false, @@ -884,7 +884,7 @@ Array [ "dependencies": Array [ "b (upstream-1)", ], - "name": "c;_phase:upstream-2", + "name": "c (upstream-2)", "silent": true, }, Object { @@ -901,40 +901,40 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:upstream-1-self", + "b (upstream-1-self)", ], - "name": "c;_phase:upstream-1-self-upstream", + "name": "c (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ "b (upstream-1)", ], - "name": "b;_phase:upstream-1-self", + "name": "b (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "c;_phase:upstream-2", + "c (upstream-2)", ], - "name": "c;_phase:upstream-2-self", + "name": "c (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ "f (upstream-3)", - "a;_phase:upstream-1-self-upstream", - "h;_phase:upstream-1-self-upstream", - "a;_phase:upstream-2-self", - "h;_phase:upstream-2-self", + "a (upstream-1-self-upstream)", + "h (upstream-1-self-upstream)", + "a (upstream-2-self)", + "h (upstream-2-self)", ], "name": "f (complex)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", - "h;_phase:upstream-2", + "a (upstream-2)", + "h (upstream-2)", ], "name": "f (upstream-3)", "silent": false, @@ -943,35 +943,35 @@ Array [ "dependencies": Array [ "a (upstream-1)", ], - "name": "h;_phase:upstream-2", + "name": "h (upstream-2)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-1-self", + "a (upstream-1-self)", ], - "name": "h;_phase:upstream-1-self-upstream", + "name": "h (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "h;_phase:upstream-2", + "h (upstream-2)", ], - "name": "h;_phase:upstream-2-self", + "name": "h (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ "g (upstream-3)", - "a;_phase:upstream-1-self-upstream", - "a;_phase:upstream-2-self", + "a (upstream-1-self-upstream)", + "a (upstream-2-self)", ], "name": "g (complex)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], "name": "g (upstream-3)", "silent": false, @@ -979,15 +979,15 @@ Array [ Object { "dependencies": Array [ "h (upstream-3)", - "a;_phase:upstream-1-self-upstream", - "a;_phase:upstream-2-self", + "a (upstream-1-self-upstream)", + "a (upstream-2-self)", ], "name": "h (complex)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], "name": "h (upstream-3)", "silent": false, @@ -1121,27 +1121,27 @@ exports[`PhasedOperationPlugin handles filtered phases on filtered projects 1`] Array [ Object { "dependencies": Array [ - "a;_phase:upstream-1", - "h;_phase:upstream-1", + "a (upstream-1)", + "h (upstream-1)", ], "name": "f (upstream-2)", "silent": false, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-1", + "name": "a (upstream-1)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:no-deps", + "a (no-deps)", ], - "name": "h;_phase:upstream-1", + "name": "h (upstream-1)", "silent": true, }, Object { "dependencies": Array [], - "name": "a;_phase:no-deps", + "name": "a (no-deps)", "silent": true, }, Object { @@ -1151,16 +1151,16 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:upstream-1", + "b (upstream-1)", ], "name": "c (upstream-2)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:no-deps", + "a (no-deps)", ], - "name": "b;_phase:upstream-1", + "name": "b (upstream-1)", "silent": true, }, ] @@ -1171,32 +1171,32 @@ Array [ Object { "dependencies": Array [ "f (upstream-3)", - "a;_phase:upstream-1-self-upstream", - "h;_phase:upstream-1-self-upstream", - "a;_phase:upstream-2-self", - "h;_phase:upstream-2-self", + "a (upstream-1-self-upstream)", + "h (upstream-1-self-upstream)", + "a (upstream-2-self)", + "h (upstream-2-self)", ], "name": "f (complex)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", - "h;_phase:upstream-2", + "a (upstream-2)", + "h (upstream-2)", ], "name": "f (upstream-3)", "silent": false, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-2", + "name": "a (upstream-2)", "silent": true, }, Object { "dependencies": Array [ "a (upstream-1)", ], - "name": "h;_phase:upstream-2", + "name": "h (upstream-2)", "silent": true, }, Object { @@ -1206,35 +1206,35 @@ Array [ }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-1-self-upstream", + "name": "a (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-1-self", + "a (upstream-1-self)", ], - "name": "h;_phase:upstream-1-self-upstream", + "name": "h (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ "a (upstream-1)", ], - "name": "a;_phase:upstream-1-self", + "name": "a (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], - "name": "a;_phase:upstream-2-self", + "name": "a (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ - "h;_phase:upstream-2", + "h (upstream-2)", ], - "name": "h;_phase:upstream-2-self", + "name": "h (upstream-2-self)", "silent": true, }, Object { @@ -1252,15 +1252,15 @@ Array [ Object { "dependencies": Array [ "c (upstream-3)", - "b;_phase:upstream-1-self-upstream", - "b;_phase:upstream-2-self", + "b (upstream-1-self-upstream)", + "b (upstream-2-self)", ], "name": "c (complex)", "silent": false, }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], "name": "c (upstream-3)", "silent": false, @@ -1269,27 +1269,27 @@ Array [ "dependencies": Array [ "a (upstream-1)", ], - "name": "b;_phase:upstream-2", + "name": "b (upstream-2)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-1-self", + "a (upstream-1-self)", ], - "name": "b;_phase:upstream-1-self-upstream", + "name": "b (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], - "name": "b;_phase:upstream-2-self", + "name": "b (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ "a (no-deps)", - "h;_phase:no-deps", + "h (no-deps)", ], "name": "f (upstream-1)", "silent": false, @@ -1301,19 +1301,19 @@ Array [ }, Object { "dependencies": Array [], - "name": "h;_phase:no-deps", + "name": "h (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", ], "name": "c (upstream-1)", "silent": false, }, Object { "dependencies": Array [], - "name": "b;_phase:no-deps", + "name": "b (no-deps)", "silent": true, }, Object { @@ -1339,52 +1339,52 @@ Array [ Object { "dependencies": Array [ "g (no-deps)", - "a;_phase:upstream-self", + "a (upstream-self)", ], "name": "g (upstream-self)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:no-deps", + "a (no-deps)", ], - "name": "a;_phase:upstream-self", + "name": "a (upstream-self)", "silent": true, }, Object { "dependencies": Array [], - "name": "a;_phase:no-deps", + "name": "a (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:no-deps", + "a (no-deps)", ], "name": "g (upstream-1)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:upstream-1", + "a (upstream-1)", ], "name": "g (upstream-2)", "silent": false, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-1", + "name": "a (upstream-1)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], "name": "g (upstream-3)", "silent": false, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-2", + "name": "a (upstream-2)", "silent": true, }, Object { @@ -1403,37 +1403,37 @@ Array [ }, Object { "dependencies": Array [ - "a;_phase:upstream-1-self", + "a (upstream-1-self)", ], "name": "g (upstream-1-self-upstream)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:upstream-1", + "a (upstream-1)", ], - "name": "a;_phase:upstream-1-self", + "name": "a (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ "g (upstream-3)", - "a;_phase:upstream-1-self-upstream", - "a;_phase:upstream-2-self", + "a (upstream-1-self-upstream)", + "a (upstream-2-self)", ], "name": "g (complex)", "silent": false, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-1-self-upstream", + "name": "a (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], - "name": "a;_phase:upstream-2-self", + "name": "a (upstream-2-self)", "silent": true, }, ] @@ -1460,7 +1460,7 @@ Array [ "dependencies": Array [ "f (no-deps)", "a (upstream-self)", - "h;_phase:upstream-self", + "h (upstream-self)", ], "name": "f (upstream-self)", "silent": false, @@ -1474,42 +1474,42 @@ Array [ }, Object { "dependencies": Array [ - "h;_phase:no-deps", + "h (no-deps)", "a (upstream-self)", ], - "name": "h;_phase:upstream-self", + "name": "h (upstream-self)", "silent": true, }, Object { "dependencies": Array [], - "name": "h;_phase:no-deps", + "name": "h (no-deps)", "silent": true, }, Object { "dependencies": Array [ "c (no-deps)", - "b;_phase:upstream-self", + "b (upstream-self)", ], "name": "c (upstream-self)", "silent": false, }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", "a (upstream-self)", ], - "name": "b;_phase:upstream-self", + "name": "b (upstream-self)", "silent": true, }, Object { "dependencies": Array [], - "name": "b;_phase:no-deps", + "name": "b (no-deps)", "silent": true, }, Object { "dependencies": Array [ "a (no-deps)", - "h;_phase:no-deps", + "h (no-deps)", ], "name": "f (upstream-1)", "silent": false, @@ -1521,7 +1521,7 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", ], "name": "c (upstream-1)", "silent": false, @@ -1529,7 +1529,7 @@ Array [ Object { "dependencies": Array [ "a (upstream-1)", - "h;_phase:upstream-1", + "h (upstream-1)", ], "name": "f (upstream-2)", "silent": false, @@ -1538,7 +1538,7 @@ Array [ "dependencies": Array [ "a (no-deps)", ], - "name": "h;_phase:upstream-1", + "name": "h (upstream-1)", "silent": true, }, Object { @@ -1548,7 +1548,7 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:upstream-1", + "b (upstream-1)", ], "name": "c (upstream-2)", "silent": false, @@ -1557,13 +1557,13 @@ Array [ "dependencies": Array [ "a (no-deps)", ], - "name": "b;_phase:upstream-1", + "name": "b (upstream-1)", "silent": true, }, Object { "dependencies": Array [ "a (upstream-2)", - "h;_phase:upstream-2", + "h (upstream-2)", ], "name": "f (upstream-3)", "silent": false, @@ -1572,7 +1572,7 @@ Array [ "dependencies": Array [ "a (upstream-1)", ], - "name": "h;_phase:upstream-2", + "name": "h (upstream-2)", "silent": true, }, Object { @@ -1582,7 +1582,7 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], "name": "c (upstream-3)", "silent": false, @@ -1591,7 +1591,7 @@ Array [ "dependencies": Array [ "a (upstream-1)", ], - "name": "b;_phase:upstream-2", + "name": "b (upstream-2)", "silent": true, }, Object { @@ -1639,16 +1639,16 @@ Array [ Object { "dependencies": Array [ "a (upstream-1-self)", - "h;_phase:upstream-1-self", + "h (upstream-1-self)", ], "name": "f (upstream-1-self-upstream)", "silent": false, }, Object { "dependencies": Array [ - "h;_phase:upstream-1", + "h (upstream-1)", ], - "name": "h;_phase:upstream-1-self", + "name": "h (upstream-1-self)", "silent": true, }, Object { @@ -1658,25 +1658,25 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:upstream-1-self", + "b (upstream-1-self)", ], "name": "c (upstream-1-self-upstream)", "silent": false, }, Object { "dependencies": Array [ - "b;_phase:upstream-1", + "b (upstream-1)", ], - "name": "b;_phase:upstream-1-self", + "name": "b (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ "f (upstream-3)", "a (upstream-1-self-upstream)", - "h;_phase:upstream-1-self-upstream", + "h (upstream-1-self-upstream)", "a (upstream-2-self)", - "h;_phase:upstream-2-self", + "h (upstream-2-self)", ], "name": "f (complex)", "silent": false, @@ -1685,14 +1685,14 @@ Array [ "dependencies": Array [ "a (upstream-1-self)", ], - "name": "h;_phase:upstream-1-self-upstream", + "name": "h (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "h;_phase:upstream-2", + "h (upstream-2)", ], - "name": "h;_phase:upstream-2-self", + "name": "h (upstream-2-self)", "silent": true, }, Object { @@ -1705,8 +1705,8 @@ Array [ Object { "dependencies": Array [ "c (upstream-3)", - "b;_phase:upstream-1-self-upstream", - "b;_phase:upstream-2-self", + "b (upstream-1-self-upstream)", + "b (upstream-2-self)", ], "name": "c (complex)", "silent": false, @@ -1715,14 +1715,14 @@ Array [ "dependencies": Array [ "a (upstream-1-self)", ], - "name": "b;_phase:upstream-1-self-upstream", + "name": "b (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], - "name": "b;_phase:upstream-2-self", + "name": "b (upstream-2-self)", "silent": true, }, ] @@ -1732,32 +1732,32 @@ exports[`PhasedOperationPlugin handles some changed projects 1`] = ` Array [ Object { "dependencies": Array [], - "name": "a;_phase:no-deps", + "name": "a (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "b;_phase:no-deps", + "name": "b (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "c;_phase:no-deps", + "name": "c (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "d;_phase:no-deps", + "name": "d (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "e;_phase:no-deps", + "name": "e (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "f;_phase:no-deps", + "name": "f (no-deps)", "silent": true, }, Object { @@ -1767,332 +1767,332 @@ Array [ }, Object { "dependencies": Array [], - "name": "h;_phase:no-deps", + "name": "h (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "i;_phase:no-deps", + "name": "i (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "j;_phase:no-deps", + "name": "j (no-deps)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:no-deps", + "a (no-deps)", ], - "name": "a;_phase:upstream-self", + "name": "a (upstream-self)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:no-deps", - "a;_phase:upstream-self", + "b (no-deps)", + "a (upstream-self)", ], - "name": "b;_phase:upstream-self", + "name": "b (upstream-self)", "silent": true, }, Object { "dependencies": Array [ - "c;_phase:no-deps", - "b;_phase:upstream-self", + "c (no-deps)", + "b (upstream-self)", ], - "name": "c;_phase:upstream-self", + "name": "c (upstream-self)", "silent": true, }, Object { "dependencies": Array [ - "d;_phase:no-deps", - "b;_phase:upstream-self", + "d (no-deps)", + "b (upstream-self)", ], - "name": "d;_phase:upstream-self", + "name": "d (upstream-self)", "silent": true, }, Object { "dependencies": Array [ - "e;_phase:no-deps", - "c;_phase:upstream-self", + "e (no-deps)", + "c (upstream-self)", ], - "name": "e;_phase:upstream-self", + "name": "e (upstream-self)", "silent": true, }, Object { "dependencies": Array [ - "f;_phase:no-deps", - "a;_phase:upstream-self", - "h;_phase:upstream-self", + "f (no-deps)", + "a (upstream-self)", + "h (upstream-self)", ], - "name": "f;_phase:upstream-self", + "name": "f (upstream-self)", "silent": true, }, Object { "dependencies": Array [ - "h;_phase:no-deps", - "a;_phase:upstream-self", + "h (no-deps)", + "a (upstream-self)", ], - "name": "h;_phase:upstream-self", + "name": "h (upstream-self)", "silent": true, }, Object { "dependencies": Array [ "g (no-deps)", - "a;_phase:upstream-self", + "a (upstream-self)", ], "name": "g (upstream-self)", "silent": false, }, Object { "dependencies": Array [ - "i;_phase:no-deps", + "i (no-deps)", ], - "name": "i;_phase:upstream-self", + "name": "i (upstream-self)", "silent": true, }, Object { "dependencies": Array [ - "j;_phase:no-deps", + "j (no-deps)", ], - "name": "j;_phase:upstream-self", + "name": "j (upstream-self)", "silent": true, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-1", + "name": "a (upstream-1)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:no-deps", + "a (no-deps)", ], - "name": "b;_phase:upstream-1", + "name": "b (upstream-1)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", ], - "name": "c;_phase:upstream-1", + "name": "c (upstream-1)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", ], - "name": "d;_phase:upstream-1", + "name": "d (upstream-1)", "silent": true, }, Object { "dependencies": Array [ - "c;_phase:no-deps", + "c (no-deps)", ], - "name": "e;_phase:upstream-1", + "name": "e (upstream-1)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:no-deps", - "h;_phase:no-deps", + "a (no-deps)", + "h (no-deps)", ], - "name": "f;_phase:upstream-1", + "name": "f (upstream-1)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:no-deps", + "a (no-deps)", ], "name": "g (upstream-1)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:no-deps", + "a (no-deps)", ], - "name": "h;_phase:upstream-1", + "name": "h (upstream-1)", "silent": true, }, Object { "dependencies": Array [], - "name": "i;_phase:upstream-1", + "name": "i (upstream-1)", "silent": true, }, Object { "dependencies": Array [], - "name": "j;_phase:upstream-1", + "name": "j (upstream-1)", "silent": true, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-2", + "name": "a (upstream-2)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-1", + "a (upstream-1)", ], - "name": "b;_phase:upstream-2", + "name": "b (upstream-2)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-1", + "b (upstream-1)", ], - "name": "c;_phase:upstream-2", + "name": "c (upstream-2)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-1", + "b (upstream-1)", ], - "name": "d;_phase:upstream-2", + "name": "d (upstream-2)", "silent": true, }, Object { "dependencies": Array [ - "c;_phase:upstream-1", + "c (upstream-1)", ], - "name": "e;_phase:upstream-2", + "name": "e (upstream-2)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-1", - "h;_phase:upstream-1", + "a (upstream-1)", + "h (upstream-1)", ], - "name": "f;_phase:upstream-2", + "name": "f (upstream-2)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-1", + "a (upstream-1)", ], "name": "g (upstream-2)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:upstream-1", + "a (upstream-1)", ], - "name": "h;_phase:upstream-2", + "name": "h (upstream-2)", "silent": true, }, Object { "dependencies": Array [], - "name": "i;_phase:upstream-2", + "name": "i (upstream-2)", "silent": true, }, Object { "dependencies": Array [], - "name": "j;_phase:upstream-2", + "name": "j (upstream-2)", "silent": true, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-3", + "name": "a (upstream-3)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], - "name": "b;_phase:upstream-3", + "name": "b (upstream-3)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], - "name": "c;_phase:upstream-3", + "name": "c (upstream-3)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], - "name": "d;_phase:upstream-3", + "name": "d (upstream-3)", "silent": true, }, Object { "dependencies": Array [ - "c;_phase:upstream-2", + "c (upstream-2)", ], - "name": "e;_phase:upstream-3", + "name": "e (upstream-3)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", - "h;_phase:upstream-2", + "a (upstream-2)", + "h (upstream-2)", ], - "name": "f;_phase:upstream-3", + "name": "f (upstream-3)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], "name": "g (upstream-3)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], - "name": "h;_phase:upstream-3", + "name": "h (upstream-3)", "silent": true, }, Object { "dependencies": Array [], - "name": "i;_phase:upstream-3", + "name": "i (upstream-3)", "silent": true, }, Object { "dependencies": Array [], - "name": "j;_phase:upstream-3", + "name": "j (upstream-3)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-1", + "a (upstream-1)", ], - "name": "a;_phase:upstream-1-self", + "name": "a (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-1", + "b (upstream-1)", ], - "name": "b;_phase:upstream-1-self", + "name": "b (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "c;_phase:upstream-1", + "c (upstream-1)", ], - "name": "c;_phase:upstream-1-self", + "name": "c (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "d;_phase:upstream-1", + "d (upstream-1)", ], - "name": "d;_phase:upstream-1-self", + "name": "d (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "e;_phase:upstream-1", + "e (upstream-1)", ], - "name": "e;_phase:upstream-1-self", + "name": "e (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "f;_phase:upstream-1", + "f (upstream-1)", ], - "name": "f;_phase:upstream-1-self", + "name": "f (upstream-1-self)", "silent": true, }, Object { @@ -2104,65 +2104,65 @@ Array [ }, Object { "dependencies": Array [ - "h;_phase:upstream-1", + "h (upstream-1)", ], - "name": "h;_phase:upstream-1-self", + "name": "h (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "i;_phase:upstream-1", + "i (upstream-1)", ], - "name": "i;_phase:upstream-1-self", + "name": "i (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "j;_phase:upstream-1", + "j (upstream-1)", ], - "name": "j;_phase:upstream-1-self", + "name": "j (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-2", + "a (upstream-2)", ], - "name": "a;_phase:upstream-2-self", + "name": "a (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], - "name": "b;_phase:upstream-2-self", + "name": "b (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ - "c;_phase:upstream-2", + "c (upstream-2)", ], - "name": "c;_phase:upstream-2-self", + "name": "c (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ - "d;_phase:upstream-2", + "d (upstream-2)", ], - "name": "d;_phase:upstream-2-self", + "name": "d (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ - "e;_phase:upstream-2", + "e (upstream-2)", ], - "name": "e;_phase:upstream-2-self", + "name": "e (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ - "f;_phase:upstream-2", + "f (upstream-2)", ], - "name": "f;_phase:upstream-2-self", + "name": "f (upstream-2-self)", "silent": true, }, Object { @@ -2174,174 +2174,174 @@ Array [ }, Object { "dependencies": Array [ - "h;_phase:upstream-2", + "h (upstream-2)", ], - "name": "h;_phase:upstream-2-self", + "name": "h (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ - "i;_phase:upstream-2", + "i (upstream-2)", ], - "name": "i;_phase:upstream-2-self", + "name": "i (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ - "j;_phase:upstream-2", + "j (upstream-2)", ], - "name": "j;_phase:upstream-2-self", + "name": "j (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [], - "name": "a;_phase:upstream-1-self-upstream", + "name": "a (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-1-self", + "a (upstream-1-self)", ], - "name": "b;_phase:upstream-1-self-upstream", + "name": "b (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-1-self", + "b (upstream-1-self)", ], - "name": "c;_phase:upstream-1-self-upstream", + "name": "c (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-1-self", + "b (upstream-1-self)", ], - "name": "d;_phase:upstream-1-self-upstream", + "name": "d (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "c;_phase:upstream-1-self", + "c (upstream-1-self)", ], - "name": "e;_phase:upstream-1-self-upstream", + "name": "e (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-1-self", - "h;_phase:upstream-1-self", + "a (upstream-1-self)", + "h (upstream-1-self)", ], - "name": "f;_phase:upstream-1-self-upstream", + "name": "f (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-1-self", + "a (upstream-1-self)", ], "name": "g (upstream-1-self-upstream)", "silent": false, }, Object { "dependencies": Array [ - "a;_phase:upstream-1-self", + "a (upstream-1-self)", ], - "name": "h;_phase:upstream-1-self-upstream", + "name": "h (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [], - "name": "i;_phase:upstream-1-self-upstream", + "name": "i (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [], - "name": "j;_phase:upstream-1-self-upstream", + "name": "j (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "a;_phase:upstream-3", + "a (upstream-3)", ], - "name": "a;_phase:complex", + "name": "a (complex)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-3", - "a;_phase:upstream-1-self-upstream", - "a;_phase:upstream-2-self", + "b (upstream-3)", + "a (upstream-1-self-upstream)", + "a (upstream-2-self)", ], - "name": "b;_phase:complex", + "name": "b (complex)", "silent": true, }, Object { "dependencies": Array [ - "c;_phase:upstream-3", - "b;_phase:upstream-1-self-upstream", - "b;_phase:upstream-2-self", + "c (upstream-3)", + "b (upstream-1-self-upstream)", + "b (upstream-2-self)", ], - "name": "c;_phase:complex", + "name": "c (complex)", "silent": true, }, Object { "dependencies": Array [ - "d;_phase:upstream-3", - "b;_phase:upstream-1-self-upstream", - "b;_phase:upstream-2-self", + "d (upstream-3)", + "b (upstream-1-self-upstream)", + "b (upstream-2-self)", ], - "name": "d;_phase:complex", + "name": "d (complex)", "silent": true, }, Object { "dependencies": Array [ - "e;_phase:upstream-3", - "c;_phase:upstream-1-self-upstream", - "c;_phase:upstream-2-self", + "e (upstream-3)", + "c (upstream-1-self-upstream)", + "c (upstream-2-self)", ], - "name": "e;_phase:complex", + "name": "e (complex)", "silent": true, }, Object { "dependencies": Array [ - "f;_phase:upstream-3", - "a;_phase:upstream-1-self-upstream", - "h;_phase:upstream-1-self-upstream", - "a;_phase:upstream-2-self", - "h;_phase:upstream-2-self", + "f (upstream-3)", + "a (upstream-1-self-upstream)", + "h (upstream-1-self-upstream)", + "a (upstream-2-self)", + "h (upstream-2-self)", ], - "name": "f;_phase:complex", + "name": "f (complex)", "silent": true, }, Object { "dependencies": Array [ "g (upstream-3)", - "a;_phase:upstream-1-self-upstream", - "a;_phase:upstream-2-self", + "a (upstream-1-self-upstream)", + "a (upstream-2-self)", ], "name": "g (complex)", "silent": false, }, Object { "dependencies": Array [ - "h;_phase:upstream-3", - "a;_phase:upstream-1-self-upstream", - "a;_phase:upstream-2-self", + "h (upstream-3)", + "a (upstream-1-self-upstream)", + "a (upstream-2-self)", ], - "name": "h;_phase:complex", + "name": "h (complex)", "silent": true, }, Object { "dependencies": Array [ - "i;_phase:upstream-3", + "i (upstream-3)", ], - "name": "i;_phase:complex", + "name": "i (complex)", "silent": true, }, Object { "dependencies": Array [ - "j;_phase:upstream-3", + "j (upstream-3)", ], - "name": "j;_phase:complex", + "name": "j (complex)", "silent": true, }, ] @@ -2356,7 +2356,7 @@ Array [ }, Object { "dependencies": Array [], - "name": "b;_phase:no-deps", + "name": "b (no-deps)", "silent": true, }, Object { @@ -2366,12 +2366,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "d;_phase:no-deps", + "name": "d (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "e;_phase:no-deps", + "name": "e (no-deps)", "silent": true, }, Object { @@ -2381,22 +2381,22 @@ Array [ }, Object { "dependencies": Array [], - "name": "g;_phase:no-deps", + "name": "g (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "h;_phase:no-deps", + "name": "h (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "i;_phase:no-deps", + "name": "i (no-deps)", "silent": true, }, Object { "dependencies": Array [], - "name": "j;_phase:no-deps", + "name": "j (no-deps)", "silent": true, }, Object { @@ -2408,7 +2408,7 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", "a (upstream-self)", ], "name": "b (upstream-self)", @@ -2424,7 +2424,7 @@ Array [ }, Object { "dependencies": Array [ - "d;_phase:no-deps", + "d (no-deps)", "b (upstream-self)", ], "name": "d (upstream-self)", @@ -2432,7 +2432,7 @@ Array [ }, Object { "dependencies": Array [ - "e;_phase:no-deps", + "e (no-deps)", "c (upstream-self)", ], "name": "e (upstream-self)", @@ -2449,7 +2449,7 @@ Array [ }, Object { "dependencies": Array [ - "h;_phase:no-deps", + "h (no-deps)", "a (upstream-self)", ], "name": "h (upstream-self)", @@ -2457,7 +2457,7 @@ Array [ }, Object { "dependencies": Array [ - "g;_phase:no-deps", + "g (no-deps)", "a (upstream-self)", ], "name": "g (upstream-self)", @@ -2465,16 +2465,16 @@ Array [ }, Object { "dependencies": Array [ - "i;_phase:no-deps", + "i (no-deps)", ], - "name": "i;_phase:upstream-self", + "name": "i (upstream-self)", "silent": true, }, Object { "dependencies": Array [ - "j;_phase:no-deps", + "j (no-deps)", ], - "name": "j;_phase:upstream-self", + "name": "j (upstream-self)", "silent": true, }, Object { @@ -2491,16 +2491,16 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", ], "name": "c (upstream-1)", "silent": false, }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", ], - "name": "d;_phase:upstream-1", + "name": "d (upstream-1)", "silent": true, }, Object { @@ -2513,7 +2513,7 @@ Array [ Object { "dependencies": Array [ "a (no-deps)", - "h;_phase:no-deps", + "h (no-deps)", ], "name": "f (upstream-1)", "silent": false, @@ -2534,12 +2534,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "i;_phase:upstream-1", + "name": "i (upstream-1)", "silent": true, }, Object { "dependencies": Array [], - "name": "j;_phase:upstream-1", + "name": "j (upstream-1)", "silent": true, }, Object { @@ -2599,12 +2599,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "i;_phase:upstream-2", + "name": "i (upstream-2)", "silent": true, }, Object { "dependencies": Array [], - "name": "j;_phase:upstream-2", + "name": "j (upstream-2)", "silent": true, }, Object { @@ -2664,12 +2664,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "i;_phase:upstream-3", + "name": "i (upstream-3)", "silent": true, }, Object { "dependencies": Array [], - "name": "j;_phase:upstream-3", + "name": "j (upstream-3)", "silent": true, }, Object { @@ -2695,9 +2695,9 @@ Array [ }, Object { "dependencies": Array [ - "d;_phase:upstream-1", + "d (upstream-1)", ], - "name": "d;_phase:upstream-1-self", + "name": "d (upstream-1-self)", "silent": true, }, Object { @@ -2730,16 +2730,16 @@ Array [ }, Object { "dependencies": Array [ - "i;_phase:upstream-1", + "i (upstream-1)", ], - "name": "i;_phase:upstream-1-self", + "name": "i (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ - "j;_phase:upstream-1", + "j (upstream-1)", ], - "name": "j;_phase:upstream-1-self", + "name": "j (upstream-1-self)", "silent": true, }, Object { @@ -2800,16 +2800,16 @@ Array [ }, Object { "dependencies": Array [ - "i;_phase:upstream-2", + "i (upstream-2)", ], - "name": "i;_phase:upstream-2-self", + "name": "i (upstream-2-self)", "silent": true, }, Object { "dependencies": Array [ - "j;_phase:upstream-2", + "j (upstream-2)", ], - "name": "j;_phase:upstream-2-self", + "name": "j (upstream-2-self)", "silent": true, }, Object { @@ -2869,12 +2869,12 @@ Array [ }, Object { "dependencies": Array [], - "name": "i;_phase:upstream-1-self-upstream", + "name": "i (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [], - "name": "j;_phase:upstream-1-self-upstream", + "name": "j (upstream-1-self-upstream)", "silent": true, }, Object { @@ -2951,16 +2951,16 @@ Array [ }, Object { "dependencies": Array [ - "i;_phase:upstream-3", + "i (upstream-3)", ], - "name": "i;_phase:complex", + "name": "i (complex)", "silent": true, }, Object { "dependencies": Array [ - "j;_phase:upstream-3", + "j (upstream-3)", ], - "name": "j;_phase:complex", + "name": "j (complex)", "silent": true, }, ] @@ -2970,7 +2970,7 @@ exports[`PhasedOperationPlugin handles some changed projects within filtered pro Array [ Object { "dependencies": Array [], - "name": "f;_phase:no-deps", + "name": "f (no-deps)", "silent": true, }, Object { @@ -2985,9 +2985,9 @@ Array [ }, Object { "dependencies": Array [ - "f;_phase:no-deps", + "f (no-deps)", "a (upstream-self)", - "h;_phase:upstream-self", + "h (upstream-self)", ], "name": "f (upstream-self)", "silent": false, @@ -3001,42 +3001,42 @@ Array [ }, Object { "dependencies": Array [ - "h;_phase:no-deps", + "h (no-deps)", "a (upstream-self)", ], - "name": "h;_phase:upstream-self", + "name": "h (upstream-self)", "silent": true, }, Object { "dependencies": Array [], - "name": "h;_phase:no-deps", + "name": "h (no-deps)", "silent": true, }, Object { "dependencies": Array [ "c (no-deps)", - "b;_phase:upstream-self", + "b (upstream-self)", ], "name": "c (upstream-self)", "silent": false, }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", "a (upstream-self)", ], - "name": "b;_phase:upstream-self", + "name": "b (upstream-self)", "silent": true, }, Object { "dependencies": Array [], - "name": "b;_phase:no-deps", + "name": "b (no-deps)", "silent": true, }, Object { "dependencies": Array [ "a (no-deps)", - "h;_phase:no-deps", + "h (no-deps)", ], "name": "f (upstream-1)", "silent": false, @@ -3048,7 +3048,7 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:no-deps", + "b (no-deps)", ], "name": "c (upstream-1)", "silent": false, @@ -3056,7 +3056,7 @@ Array [ Object { "dependencies": Array [ "a (upstream-1)", - "h;_phase:upstream-1", + "h (upstream-1)", ], "name": "f (upstream-2)", "silent": false, @@ -3065,7 +3065,7 @@ Array [ "dependencies": Array [ "a (no-deps)", ], - "name": "h;_phase:upstream-1", + "name": "h (upstream-1)", "silent": true, }, Object { @@ -3075,7 +3075,7 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:upstream-1", + "b (upstream-1)", ], "name": "c (upstream-2)", "silent": false, @@ -3084,13 +3084,13 @@ Array [ "dependencies": Array [ "a (no-deps)", ], - "name": "b;_phase:upstream-1", + "name": "b (upstream-1)", "silent": true, }, Object { "dependencies": Array [ "a (upstream-2)", - "h;_phase:upstream-2", + "h (upstream-2)", ], "name": "f (upstream-3)", "silent": false, @@ -3099,7 +3099,7 @@ Array [ "dependencies": Array [ "a (upstream-1)", ], - "name": "h;_phase:upstream-2", + "name": "h (upstream-2)", "silent": true, }, Object { @@ -3109,7 +3109,7 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], "name": "c (upstream-3)", "silent": false, @@ -3118,7 +3118,7 @@ Array [ "dependencies": Array [ "a (upstream-1)", ], - "name": "b;_phase:upstream-2", + "name": "b (upstream-2)", "silent": true, }, Object { @@ -3166,16 +3166,16 @@ Array [ Object { "dependencies": Array [ "a (upstream-1-self)", - "h;_phase:upstream-1-self", + "h (upstream-1-self)", ], "name": "f (upstream-1-self-upstream)", "silent": false, }, Object { "dependencies": Array [ - "h;_phase:upstream-1", + "h (upstream-1)", ], - "name": "h;_phase:upstream-1-self", + "name": "h (upstream-1-self)", "silent": true, }, Object { @@ -3185,25 +3185,25 @@ Array [ }, Object { "dependencies": Array [ - "b;_phase:upstream-1-self", + "b (upstream-1-self)", ], "name": "c (upstream-1-self-upstream)", "silent": false, }, Object { "dependencies": Array [ - "b;_phase:upstream-1", + "b (upstream-1)", ], - "name": "b;_phase:upstream-1-self", + "name": "b (upstream-1-self)", "silent": true, }, Object { "dependencies": Array [ "f (upstream-3)", "a (upstream-1-self-upstream)", - "h;_phase:upstream-1-self-upstream", + "h (upstream-1-self-upstream)", "a (upstream-2-self)", - "h;_phase:upstream-2-self", + "h (upstream-2-self)", ], "name": "f (complex)", "silent": false, @@ -3212,14 +3212,14 @@ Array [ "dependencies": Array [ "a (upstream-1-self)", ], - "name": "h;_phase:upstream-1-self-upstream", + "name": "h (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "h;_phase:upstream-2", + "h (upstream-2)", ], - "name": "h;_phase:upstream-2-self", + "name": "h (upstream-2-self)", "silent": true, }, Object { @@ -3232,8 +3232,8 @@ Array [ Object { "dependencies": Array [ "c (upstream-3)", - "b;_phase:upstream-1-self-upstream", - "b;_phase:upstream-2-self", + "b (upstream-1-self-upstream)", + "b (upstream-2-self)", ], "name": "c (complex)", "silent": false, @@ -3242,14 +3242,14 @@ Array [ "dependencies": Array [ "a (upstream-1-self)", ], - "name": "b;_phase:upstream-1-self-upstream", + "name": "b (upstream-1-self-upstream)", "silent": true, }, Object { "dependencies": Array [ - "b;_phase:upstream-2", + "b (upstream-2)", ], - "name": "b;_phase:upstream-2-self", + "name": "b (upstream-2-self)", "silent": true, }, ] diff --git a/rush-plugins/rush-serve-plugin/src/api.types.ts b/rush-plugins/rush-serve-plugin/src/api.types.ts index 40497b4bc7b..7509c7fa50b 100644 --- a/rush-plugins/rush-serve-plugin/src/api.types.ts +++ b/rush-plugins/rush-serve-plugin/src/api.types.ts @@ -44,10 +44,17 @@ export interface IOperationInfo { */ phaseName: string; + /** + * If false, this operation is disabled and will/did not execute during the current run. + * The status will be reported as `Skipped`. + */ + enabled: boolean; + /** * If true, this operation is configured to be silent and is included for completeness. */ silent: boolean; + /** * If true, this operation is configured to be a noop and is included for graph completeness. */ diff --git a/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts b/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts index 60682d6fb00..162172d325b 100644 --- a/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts +++ b/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts @@ -325,7 +325,7 @@ function tryEnableBuildStatusWebSocketServer( */ function convertToOperationInfo(record: IOperationExecutionResult): IOperationInfo | undefined { const { operation } = record; - const { name, associatedPhase, associatedProject, runner } = operation; + const { name, associatedPhase, associatedProject, runner, enabled } = operation; if (!name || !associatedPhase || !associatedProject || !runner) { return; @@ -338,7 +338,8 @@ function tryEnableBuildStatusWebSocketServer( packageName, phaseName: associatedPhase.name, - silent: !!runner.silent, + enabled, + silent: record.silent, noop: !!runner.isNoOp, status: readableStatusFromStatus[record.status], From f5c875faa8266119b4a7d8baaea48eb3089aec64 Mon Sep 17 00:00:00 2001 From: David Michon Date: Fri, 4 Oct 2024 21:04:03 +0000 Subject: [PATCH 6/6] [rush] Add `cacheHashSalt` property --- ...split-project-change-analyzer_2024-10-04-21-03.json | 10 ++++++++++ common/reviews/api/rush-lib.api.md | 1 + .../rush-init/common/config/rush/build-cache.json | 5 +++++ libraries/rush-lib/src/api/BuildCacheConfiguration.ts | 9 +++++++++ .../src/logic/operations/CacheableOperationPlugin.ts | 8 ++++++++ libraries/rush-lib/src/schemas/build-cache.schema.json | 4 ++++ 6 files changed, 37 insertions(+) create mode 100644 common/changes/@microsoft/rush/split-project-change-analyzer_2024-10-04-21-03.json diff --git a/common/changes/@microsoft/rush/split-project-change-analyzer_2024-10-04-21-03.json b/common/changes/@microsoft/rush/split-project-change-analyzer_2024-10-04-21-03.json new file mode 100644 index 00000000000..084a641cd74 --- /dev/null +++ b/common/changes/@microsoft/rush/split-project-change-analyzer_2024-10-04-21-03.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Add an optional property `cacheHashSalt` to `build-cache.json` to allow repository maintainers to globally force a hash change in build cache entries.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 8ea631c00c6..e1747b30ca5 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -66,6 +66,7 @@ export class ApprovedPackagesPolicy { // @beta export class BuildCacheConfiguration { readonly buildCacheEnabled: boolean; + readonly cacheHashSalt: string | undefined; cacheWriteEnabled: boolean; readonly cloudCacheProvider: ICloudBuildCacheProvider | undefined; static getBuildCacheConfigFilePath(rushConfiguration: RushConfiguration): string; diff --git a/libraries/rush-lib/assets/rush-init/common/config/rush/build-cache.json b/libraries/rush-lib/assets/rush-init/common/config/rush/build-cache.json index 959c69d4ba2..95c6e272202 100644 --- a/libraries/rush-lib/assets/rush-init/common/config/rush/build-cache.json +++ b/libraries/rush-lib/assets/rush-init/common/config/rush/build-cache.json @@ -34,6 +34,11 @@ */ // "cacheEntryNamePattern": "[projectName:normalize]-[phaseName:normalize]-[hash]" + /** + * (Optional) Salt to inject during calculation of the cache key. This can be used to invalidate the cache for all projects when the salt changes. + */ + // "cacheHashSalt": "1", + /** * Use this configuration with "cacheProvider"="azure-blob-storage" */ diff --git a/libraries/rush-lib/src/api/BuildCacheConfiguration.ts b/libraries/rush-lib/src/api/BuildCacheConfiguration.ts index c90a1aecce4..57c47661b33 100644 --- a/libraries/rush-lib/src/api/BuildCacheConfiguration.ts +++ b/libraries/rush-lib/src/api/BuildCacheConfiguration.ts @@ -42,6 +42,10 @@ export interface IBaseBuildCacheJson { * The token parser is in CacheEntryId.ts */ cacheEntryNamePattern?: string; + /** + * An optional salt to inject during calculation of the cache key. This can be used to invalidate the cache for all projects when the salt changes. + */ + cacheHashSalt?: string; } /** @@ -102,6 +106,10 @@ export class BuildCacheConfiguration { * The provider for interacting with the cloud build cache, if configured. */ public readonly cloudCacheProvider: ICloudBuildCacheProvider | undefined; + /** + * An optional salt to inject during calculation of the cache key. This can be used to invalidate the cache for all projects when the salt changes. + */ + public readonly cacheHashSalt: string | undefined; private constructor({ getCacheEntryId, @@ -120,6 +128,7 @@ export class BuildCacheConfiguration { rushConfiguration: rushConfiguration }); this.cloudCacheProvider = cloudCacheProvider; + this.cacheHashSalt = buildCacheJson.cacheHashSalt; } /** diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 270d82dc2c3..5632236f735 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -88,6 +88,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { const { allowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration } = this._options; + const { cacheHashSalt } = buildCacheConfiguration; + hooks.beforeExecuteOperations.tap( PLUGIN_NAME, ( @@ -143,6 +145,12 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // of the build cache. hasher.update(`${RushConstants.buildCacheVersion}`); + if (cacheHashSalt !== undefined) { + // This allows repository owners to force a cache bust by changing the salt. + // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. + hasher.update(cacheHashSalt); + } + for (const dependencyHash of dependencyHashes) { hasher.update(dependencyHash); } diff --git a/libraries/rush-lib/src/schemas/build-cache.schema.json b/libraries/rush-lib/src/schemas/build-cache.schema.json index 4ec92204e5d..65962422b19 100644 --- a/libraries/rush-lib/src/schemas/build-cache.schema.json +++ b/libraries/rush-lib/src/schemas/build-cache.schema.json @@ -33,6 +33,10 @@ "type": "string", "description": "Setting this property overrides the cache entry ID. If this property is set, it must contain a [hash] token. It may also contain one of the following tokens: [projectName], [projectName:normalize], [phaseName], [phaseName:normalize], [phaseName:trimPrefix], [os], and [arch]." }, + "cacheHashSalt": { + "type": "string", + "description": "An optional salt to inject during calculation of the cache key. This can be used to invalidate the cache for all projects when the salt changes." + }, "azureBlobStorageConfiguration": { "type": "object", "additionalProperties": false,