[rush] Split ProjectChangeAnalyzer, fix build cache hashes

build-tests/heft-node-everything-test/config/heft.json

@@ -26,6 +26,16 @@
           "taskPlugin": {
             "pluginPackage": "@rushstack/heft-api-extractor-plugin"
           }
+        },
+        "metadata-test": {
+          "taskDependencies": ["typescript"],
+          "taskPlugin": {
+            "pluginPackage": "@rushstack/heft",
+            "pluginName": "run-script-plugin",
+            "options": {
+              "scriptPath": "./lib/test-metadata.js"
+            }
+          }
         }
       }
     },

build-tests/heft-node-everything-test/package.json

@@ -8,7 +8,9 @@
   "scripts": {
     "build": "heft build --clean",
     "_phase:build": "heft run --only build -- --clean",
-    "_phase:test": "heft run --only test -- --clean"
+    "_phase:build:incremental": "heft run --only build --",
+    "_phase:test": "heft run --only test -- --clean",
+    "_phase:test:incremental": "heft run --only test --"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "workspace:*",

build-tests/heft-node-everything-test/src/test-metadata.ts

@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import * as fs from 'node:fs/promises';
+
+import type { IRunScriptOptions } from '@rushstack/heft';
+
+export async function runAsync({ heftConfiguration: { buildFolderPath } }: IRunScriptOptions): Promise<void> {
+  const metadataFolder: string = `${buildFolderPath}/.rush/temp/operation/_phase_build`;
+
+  await fs.mkdir(metadataFolder, { recursive: true });
+
+  await fs.writeFile(`${metadataFolder}/test.txt`, new Date().toString(), 'utf-8');
+}

common/changes/@microsoft/rush/metadata-folder_2024-10-03-23-13.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Allow rush plugins to extend build cache entries by writing additional files to the metadata folder. Expose the metadata folder path to plugins.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-06.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "[CACHE BREAK] Alter the computation of build cache IDs to depend on the graph of operations in the build and therefore account for multiple phases, rather than only the declared dependencies. Ensure that `dependsOnEnvVars` and command line parameters that affect upstream phases impact the cache IDs of downstream operations.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/changes/@microsoft/rush/split-project-change-analyzer_2024-01-09-22-11.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "(BREAKING CHANGE) Replace use of `ProjectChangeAnalyzer` in phased command hooks with a new `InputsSnapshot` data structure that is completely synchronous and does not perform any disk operations. Perform all disk operations and state computation prior to executing the build graph.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/changes/@microsoft/rush/split-project-change-analyzer_2024-10-04-00-55.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Add a new property `enabled` to `Operation` that when set to false, will cause the execution engine to immediately return `OperationStatus.Skipped` instead of invoking the runner. Use this property to disable operations that are not intended to be executed in the current pass, e.g. those that did not contain changes in the most recent watch iteration, or those excluded by `--only`.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/changes/@microsoft/rush/split-project-change-analyzer_2024-10-04-21-03.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Add an optional property `cacheHashSalt` to `build-cache.json` to allow repository maintainers to globally force a hash change in build cache entries.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/changes/@rushstack/lookup-by-path/split-project-change-analyzer_2024-09-24-23-54.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/lookup-by-path",
+      "comment": "Add `IReadonlyLookupByPath` interface to help unit tests for functions that consume `LookupByPath`.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@rushstack/lookup-by-path"
+}

common/reviews/api/lookup-by-path.api.md

@@ -12,7 +12,15 @@ export interface IPrefixMatch<TItem> {
 }
 
 // @beta
-export class LookupByPath<TItem> {
+export interface IReadonlyLookupByPath<TItem> {
+    findChildPath(childPath: string): TItem | undefined;
+    findChildPathFromSegments(childPathSegments: Iterable<string>): TItem | undefined;
+    findLongestPrefixMatch(query: string): IPrefixMatch<TItem> | undefined;
+    groupByChild<TInfo>(infoByPath: Map<string, TInfo>): Map<TItem, Map<string, TInfo>>;
+}
+
+// @beta
+export class LookupByPath<TItem> implements IReadonlyLookupByPath<TItem> {
     constructor(entries?: Iterable<[string, TItem]>, delimiter?: string);
     readonly delimiter: string;
     findChildPath(childPath: string): TItem | undefined;

common/reviews/api/rush-lib.api.md

@@ -66,6 +66,7 @@ export class ApprovedPackagesPolicy {
 // @beta
 export class BuildCacheConfiguration {
     readonly buildCacheEnabled: boolean;
+    readonly cacheHashSalt: string | undefined;
     cacheWriteEnabled: boolean;
     readonly cloudCacheProvider: ICloudBuildCacheProvider | undefined;
     static getBuildCacheConfigFilePath(rushConfiguration: RushConfiguration): string;
@@ -330,6 +331,9 @@ export class _FlagFile<TState extends JsonObject = JsonObject> {
 // @beta
 export type GetCacheEntryIdFunction = (options: IGenerateCacheEntryIdOptions) => string;
 
+// @beta
+export type GetInputsSnapshotAsyncFn = () => Promise<IInputsSnapshot | undefined>;
+
 // @internal (undocumented)
 export interface _IBuiltInPluginConfiguration extends _IRushPluginConfigurationBase {
     // (undocumented)
@@ -462,7 +466,7 @@ export interface IEnvironmentConfigurationInitializeOptions {
 
 // @alpha
 export interface IExecuteOperationsContext extends ICreateOperationsContext {
-    readonly projectChangeAnalyzer: ProjectChangeAnalyzer;
+    readonly inputsSnapshot?: IInputsSnapshot;
 }
 
 // @alpha
@@ -521,6 +525,14 @@ export interface IGetChangedProjectsOptions {
 export interface IGlobalCommand extends IRushCommand {
 }
 
+// @beta
+export interface IInputsSnapshot {
+    getOperationOwnStateHash(project: IRushConfigurationProjectForSnapshot, operationName?: string): string;
+    getTrackedFileHashesForOperation(project: IRushConfigurationProjectForSnapshot, operationName?: string): ReadonlyMap<string, string>;
+    readonly hashes: ReadonlyMap<string, string>;
+    readonly rootDirectory: string;
+}
+
 // @public
 export interface ILaunchOptions {
     alreadyReportedNodeTooNewError?: boolean;
@@ -570,8 +582,10 @@ export interface IOperationExecutionResult {
     readonly cobuildRunnerId: string | undefined;
     readonly error: Error | undefined;
     readonly logFilePaths: ILogFilePaths | undefined;
+    readonly metadataFolderPath: string | undefined;
     readonly nonCachedDurationMs: number | undefined;
     readonly operation: Operation;
+    readonly silent: boolean;
     readonly status: OperationStatus;
     readonly stdioSummarizer: StdioSummarizer;
     readonly stopwatch: IStopwatchResult;
@@ -759,16 +773,6 @@ export interface IPnpmPeerDependencyRules {
 
 export { IPrefixMatch }
 
-// @internal (undocumented)
-export interface _IRawRepoState {
-    // (undocumented)
-    projectState: Map<RushConfigurationProject, Map<string, string>> | undefined;
-    // (undocumented)
-    rawHashes: Map<string, string>;
-    // (undocumented)
-    rootDir: string;
-}
-
 // @beta
 export interface IRushCommand {
     readonly actionName: string;
@@ -798,6 +802,9 @@ export interface IRushCommandLineSpec {
     actions: IRushCommandLineAction[];
 }
 
+// @beta (undocumented)
+export type IRushConfigurationProjectForSnapshot = Pick<RushConfigurationProject, 'projectFolder' | 'projectRelativeFolder'>;
+
 // @alpha (undocumented)
 export interface IRushPhaseSharding {
     count: number;
@@ -924,6 +931,7 @@ export class Operation {
     readonly consumers: ReadonlySet<Operation>;
     deleteDependency(dependency: Operation): void;
     readonly dependencies: ReadonlySet<Operation>;
+    enabled: boolean;
     get isNoOp(): boolean;
     logFilenameIdentifier: string;
     get name(): string | undefined;
@@ -937,7 +945,7 @@ export class _OperationMetadataManager {
     constructor(options: _IOperationMetadataManagerOptions);
     // (undocumented)
     readonly logFilenameIdentifier: string;
-    get relativeFilepaths(): string[];
+    get metadataFolderPath(): string;
     // (undocumented)
     saveAsync({ durationInSeconds, cobuildContextId, cobuildRunnerId, logPath, errorLogPath, logChunksPath }: _IOperationMetadata): Promise<void>;
     // (undocumented)
@@ -1113,16 +1121,12 @@ export type PnpmStoreOptions = PnpmStoreLocation;
 export class ProjectChangeAnalyzer {
     constructor(rushConfiguration: RushConfiguration);
     // @internal (undocumented)
-    _ensureInitializedAsync(terminal: ITerminal): Promise<_IRawRepoState | undefined>;
-    // (undocumented)
     _filterProjectDataAsync<T>(project: RushConfigurationProject, unfilteredProjectData: Map<string, T>, rootDir: string, terminal: ITerminal): Promise<Map<string, T>>;
     getChangedProjectsAsync(options: IGetChangedProjectsOptions): Promise<Set<RushConfigurationProject>>;
     // (undocumented)
     protected getChangesByProject(lookup: LookupByPath<RushConfigurationProject>, changedFiles: Map<string, IFileDiffStatus>): Map<RushConfigurationProject, Map<string, IFileDiffStatus>>;
     // @internal
-    _tryGetProjectDependenciesAsync(project: RushConfigurationProject, terminal: ITerminal): Promise<Map<string, string> | undefined>;
-    // @internal
-    _tryGetProjectStateHashAsync(project: RushConfigurationProject, terminal: ITerminal): Promise<string | undefined>;
+    _tryGetSnapshotProviderAsync(projectConfigurations: ReadonlyMap<RushConfigurationProject, RushProjectConfiguration>, terminal: ITerminal): Promise<GetInputsSnapshotAsyncFn | undefined>;
 }
 
 // @public

libraries/lookup-by-path/src/LookupByPath.ts

@@ -46,6 +46,69 @@ export interface IPrefixMatch<TItem> {
   lastMatch?: IPrefixMatch<TItem>;
 }
 
+/**
+ * The readonly component of `LookupByPath`, to simplify unit testing.
+ *
+ * @beta
+ */
+export interface IReadonlyLookupByPath<TItem> {
+  /**
+   * Searches for the item associated with `childPath`, or the nearest ancestor of that path that
+   * has an associated item.
+   *
+   * @returns the found item, or `undefined` if no item was found
+   *
+   * @example
+   * ```ts
+   * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]);
+   * trie.findChildPath('foo/baz'); // returns 1
+   * trie.findChildPath('foo/bar/baz'); // returns 2
+   * ```
+   */
+  findChildPath(childPath: string): TItem | undefined;
+
+  /**
+   * Searches for the item for which the recorded prefix is the longest matching prefix of `query`.
+   * Obtains both the item and the length of the matched prefix, so that the remainder of the path can be
+   * extracted.
+   *
+   * @returns the found item and the length of the matched prefix, or `undefined` if no item was found
+   *
+   * @example
+   * ```ts
+   * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]);
+   * trie.findLongestPrefixMatch('foo/baz'); // returns { item: 1, index: 3 }
+   * trie.findLongestPrefixMatch('foo/bar/baz'); // returns { item: 2, index: 7 }
+   * ```
+   */
+  findLongestPrefixMatch(query: string): IPrefixMatch<TItem> | undefined;
+
+  /**
+   * Searches for the item associated with `childPathSegments`, or the nearest ancestor of that path that
+   * has an associated item.
+   *
+   * @returns the found item, or `undefined` if no item was found
+   *
+   * @example
+   * ```ts
+   * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]);
+   * trie.findChildPathFromSegments(['foo', 'baz']); // returns 1
+   * trie.findChildPathFromSegments(['foo','bar', 'baz']); // returns 2
+   * ```
+   */
+  findChildPathFromSegments(childPathSegments: Iterable<string>): TItem | undefined;
+
+  /**
+   * Groups the provided map of info by the nearest entry in the trie that contains the path. If the path
+   * is not found in the trie, the info is ignored.
+   *
+   * @returns The grouped info, grouped by the nearest entry in the trie that contains the path
+   *
+   * @param infoByPath - The info to be grouped, keyed by path
+   */
+  groupByChild<TInfo>(infoByPath: Map<string, TInfo>): Map<TItem, Map<string, TInfo>>;
+}
+
 /**
  * This class is used to associate path-like-strings, such as those returned by `git` commands,
  * with entities that correspond with ancestor folders, such as Rush Projects or npm packages.
@@ -66,7 +129,7 @@ export interface IPrefixMatch<TItem> {
  * ```
  * @beta
  */
-export class LookupByPath<TItem> {
+export class LookupByPath<TItem> implements IReadonlyLookupByPath<TItem> {
   /**
    * The delimiter used to split paths
    */
@@ -178,52 +241,21 @@ export class LookupByPath<TItem> {
   }
 
   /**
-   * Searches for the item associated with `childPath`, or the nearest ancestor of that path that
-   * has an associated item.
-   *
-   * @returns the found item, or `undefined` if no item was found
-   *
-   * @example
-   * ```ts
-   * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]);
-   * trie.findChildPath('foo/baz'); // returns 1
-   * trie.findChildPath('foo/bar/baz'); // returns 2
-   * ```
+   * {@inheritdoc IReadonlyLookupByPath}
    */
   public findChildPath(childPath: string): TItem | undefined {
     return this.findChildPathFromSegments(LookupByPath.iteratePathSegments(childPath, this.delimiter));
   }
 
   /**
-   * Searches for the item for which the recorded prefix is the longest matching prefix of `query`.
-   * Obtains both the item and the length of the matched prefix, so that the remainder of the path can be
-   * extracted.
-   *
-   * @returns the found item and the length of the matched prefix, or `undefined` if no item was found
-   *
-   * @example
-   * ```ts
-   * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]);
-   * trie.findLongestPrefixMatch('foo/baz'); // returns { item: 1, index: 3 }
-   * trie.findLongestPrefixMatch('foo/bar/baz'); // returns { item: 2, index: 7 }
-   * ```
+   * {@inheritdoc IReadonlyLookupByPath}
    */
   public findLongestPrefixMatch(query: string): IPrefixMatch<TItem> | undefined {
     return this._findLongestPrefixMatch(LookupByPath._iteratePrefixes(query, this.delimiter));
   }
 
   /**
-   * Searches for the item associated with `childPathSegments`, or the nearest ancestor of that path that
-   * has an associated item.
-   *
-   * @returns the found item, or `undefined` if no item was found
-   *
-   * @example
-   * ```ts
-   * const trie = new LookupByPath([['foo', 1], ['foo/bar', 2]]);
-   * trie.findChildPathFromSegments(['foo', 'baz']); // returns 1
-   * trie.findChildPathFromSegments(['foo','bar', 'baz']); // returns 2
-   * ```
+   * {@inheritdoc IReadonlyLookupByPath}
    */
   public findChildPathFromSegments(childPathSegments: Iterable<string>): TItem | undefined {
     let node: IPathTrieNode<TItem> = this._root;
@@ -247,12 +279,7 @@ export class LookupByPath<TItem> {
   }
 
   /**
-   * Groups the provided map of info by the nearest entry in the trie that contains the path. If the path
-   * is not found in the trie, the info is ignored.
-   *
-   * @returns The grouped info, grouped by the nearest entry in the trie that contains the path
-   *
-   * @param infoByPath - The info to be grouped, keyed by path
+   * {@inheritdoc IReadonlyLookupByPath}
    */
   public groupByChild<TInfo>(infoByPath: Map<string, TInfo>): Map<TItem, Map<string, TInfo>> {
     const groupedInfoByChild: Map<TItem, Map<string, TInfo>> = new Map();