[rush] feat(buildcache): improve access to operation build cache

common/changes/@microsoft/rush/sennyeya-cache-id_2024-12-26-23-12.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "comment": "Simplifies the process of going from operation to build cache ID.",
+      "type": "none",
+      "packageName": "@microsoft/rush"
+    }
+  ],
+  "packageName": "@microsoft/rush",
+  "email": "aramissennyeydd@users.noreply.github.com"
+}

common/reviews/api/rush-lib.api.md

@@ -931,6 +931,11 @@ export class Operation {
     addDependency(dependency: Operation): void;
     readonly associatedPhase: IPhase | undefined;
     readonly associatedProject: RushConfigurationProject | undefined;
+    // (undocumented)
+    calculateStateHash(options: {
+        inputsSnapshot: IInputsSnapshot;
+        buildCacheConfiguration: BuildCacheConfiguration;
+    }): string;
     readonly consumers: ReadonlySet<Operation>;
     deleteDependency(dependency: Operation): void;
     readonly dependencies: ReadonlySet<Operation>;
@@ -940,6 +945,8 @@ export class Operation {
     get name(): string | undefined;
     runner: IOperationRunner | undefined;
     settings: IOperationSettings | undefined;
+    // (undocumented)
+    get stateHash(): string;
     weight: number;
 }
 

libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts

@@ -13,33 +13,37 @@ import type { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider';
 import type { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider';
 import { TarExecutable } from '../../utilities/TarExecutable';
 import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration';
+import type { Operation } from '../operations/Operation';
 
-export interface IProjectBuildCacheOptions {
+export interface IOperationBuildCacheOptions {
   /**
    * The repo-wide configuration for the build cache.
    */
   buildCacheConfiguration: BuildCacheConfiguration;
-  /**
-   * The project to be cached.
-   */
-  project: RushConfigurationProject;
   /**
    * Value from rush-project.json
    */
   projectOutputFolderNames: ReadonlyArray<string>;
-  /**
-   * The hash of all relevant inputs and configuration that uniquely identifies this execution.
-   */
-  operationStateHash: string;
   /**
    * The terminal to use for logging.
    */
   terminal: ITerminal;
+}
+
+export type IProjectBuildCacheOptions = IOperationBuildCacheOptions & {
+  /**
+   * The project to be cached.
+   */
+  project: RushConfigurationProject;
+  /**
+   * The hash of all relevant inputs and configuration that uniquely identifies this execution.
+   */
+  operationStateHash: string;
   /**
    * The name of the phase that is being cached.
    */
   phaseName: string;
-}
+};
 
 interface IPathsToCache {
   filteredOutputFolderNames: string[];
@@ -94,6 +98,23 @@ export class ProjectBuildCache {
     return new ProjectBuildCache(cacheId, options);
   }
 
+  public static forOperation(operation: Operation, options: IOperationBuildCacheOptions): ProjectBuildCache {
+    if (!operation.associatedProject) {
+      throw new InternalError('Operation must have an associated project');
+    }
+    if (!operation.associatedPhase) {
+      throw new InternalError('Operation must have an associated phase');
+    }
+    const buildCacheOptions: IProjectBuildCacheOptions = {
+      ...options,
+      project: operation.associatedProject,
+      phaseName: operation.associatedPhase.name,
+      operationStateHash: operation.stateHash
+    };
+    const cacheId: string | undefined = ProjectBuildCache._getCacheId(buildCacheOptions);
+    return new ProjectBuildCache(cacheId, buildCacheOptions);
+  }
+
   public async tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise<boolean> {
     const cacheId: string | undefined = specifiedCacheId || this._cacheId;
     if (!cacheId) {

libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts

@@ -35,7 +35,6 @@ import type { IPhase } from '../../api/CommandLineConfiguration';
 import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration';
 import type { IOperationExecutionResult } from './IOperationExecutionResult';
 import type { OperationExecutionRecord } from './OperationExecutionRecord';
-import type { IInputsSnapshot } from '../incremental/InputsSnapshot';
 
 const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin';
 const PERIODIC_CALLBACK_INTERVAL_IN_SECONDS: number = 10;
@@ -88,8 +87,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
   public apply(hooks: PhasedCommandHooks): void {
     const { allowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration } = this._options;
 
-    const { cacheHashSalt } = buildCacheConfiguration;
-
     hooks.beforeExecuteOperations.tap(
       PLUGIN_NAME,
       (
@@ -104,76 +101,15 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
           );
         }
 
-        // This redefinition is necessary due to limitations in TypeScript's control flow analysis, due to the nested closure.
-        const definitelyDefinedInputsSnapshot: IInputsSnapshot = inputsSnapshot;
-
         const disjointSet: DisjointSet<Operation> | undefined = cobuildConfiguration?.cobuildFeatureEnabled
           ? new DisjointSet()
           : undefined;
 
-        const hashByOperation: Map<Operation, string> = new Map();
-        // Build cache hashes are computed up front to ensure stability and to catch configuration errors early.
-        function getOrCreateOperationHash(operation: Operation): string {
-          const cachedHash: string | undefined = hashByOperation.get(operation);
-          if (cachedHash !== undefined) {
-            return cachedHash;
-          }
-
-          // Examples of data in the config hash:
-          // - CLI parameters (ShellOperationRunner)
-          const configHash: string | undefined = operation.runner?.getConfigHash();
-
-          const { associatedProject, associatedPhase } = operation;
-          // Examples of data in the local state hash:
-          // - Environment variables specified in `dependsOnEnvVars`
-          // - Git hashes of tracked files in the associated project
-          // - Git hash of the shrinkwrap file for the project
-          // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project)
-          const localStateHash: string | undefined =
-            associatedProject &&
-            definitelyDefinedInputsSnapshot.getOperationOwnStateHash(
-              associatedProject,
-              associatedPhase?.name
-            );
-
-          // The final state hashes of operation dependencies are factored into the hash to ensure that any
-          // state changes in dependencies will invalidate the cache.
-          const dependencyHashes: string[] = Array.from(operation.dependencies, getDependencyHash).sort();
-
-          const hasher: crypto.Hash = crypto.createHash('sha1');
-          // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content
-          // of the build cache.
-          hasher.update(`${RushConstants.buildCacheVersion}`);
-
-          if (cacheHashSalt !== undefined) {
-            // This allows repository owners to force a cache bust by changing the salt.
-            // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output.
-            hasher.update(cacheHashSalt);
-          }
-
-          for (const dependencyHash of dependencyHashes) {
-            hasher.update(dependencyHash);
-          }
-
-          if (localStateHash) {
-            hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`);
-          }
-
-          if (configHash) {
-            hasher.update(`${RushConstants.hashDelimiter}${configHash}`);
-          }
-
-          const hashString: string = hasher.digest('hex');
-
-          hashByOperation.set(operation, hashString);
-          return hashString;
-        }
-
-        function getDependencyHash(operation: Operation): string {
-          return `${RushConstants.hashDelimiter}${operation.name}=${getOrCreateOperationHash(operation)}`;
-        }
-
         for (const [operation, record] of recordByOperation) {
+          const stateHash: string = operation.calculateStateHash({
+            inputsSnapshot,
+            buildCacheConfiguration
+          });
           const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation;
           if (!associatedProject || !associatedPhase || !runner) {
             return;
@@ -188,7 +124,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
           // depending on the selected phase.
           const fileHashes: ReadonlyMap<string, string> | undefined =
             inputsSnapshot.getTrackedFileHashesForOperation(associatedProject, phaseName);
-          const stateHash: string = getOrCreateOperationHash(operation);
 
           const cacheDisabledReason: string | undefined = projectConfiguration
             ? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName, operation.isNoOp)
@@ -325,10 +260,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
           let projectBuildCache: ProjectBuildCache | undefined = this._tryGetProjectBuildCache({
             buildCacheContext,
             buildCacheConfiguration,
-            rushProject: project,
-            phase,
             terminal: buildCacheTerminal,
-            operation: operation
+            operation
           });
 
           // Try to acquire the cobuild lock
@@ -639,15 +572,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
   private _tryGetProjectBuildCache({
     buildCacheConfiguration,
     buildCacheContext,
-    rushProject,
-    phase,
     terminal,
     operation
   }: {
     buildCacheContext: IOperationBuildCacheContext;
     buildCacheConfiguration: BuildCacheConfiguration | undefined;
-    rushProject: RushConfigurationProject;
-    phase: IPhase;
     terminal: ITerminal;
     operation: Operation;
   }): ProjectBuildCache | undefined {
@@ -658,20 +587,17 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
         return;
       }
 
-      const { outputFolderNames, stateHash: operationStateHash } = buildCacheContext;
+      const { outputFolderNames } = buildCacheContext;
       if (!outputFolderNames || !buildCacheConfiguration) {
         // Unreachable, since this will have set `cacheDisabledReason`.
         return;
       }
 
       // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent
-      buildCacheContext.operationBuildCache = ProjectBuildCache.getProjectBuildCache({
-        project: rushProject,
+      buildCacheContext.operationBuildCache = ProjectBuildCache.forOperation(operation, {
         projectOutputFolderNames: outputFolderNames,
         buildCacheConfiguration,
-        terminal,
-        operationStateHash,
-        phaseName: phase.name
+        terminal
       });
     }
 

libraries/rush-lib/src/logic/operations/Operation.ts

@@ -5,6 +5,10 @@ import type { RushConfigurationProject } from '../../api/RushConfigurationProjec
 import type { IPhase } from '../../api/CommandLineConfiguration';
 import type { IOperationRunner } from './IOperationRunner';
 import type { IOperationSettings } from '../../api/RushProjectConfiguration';
+import { RushConstants } from '../RushConstants';
+import * as crypto from 'crypto';
+import type { IInputsSnapshot } from '../incremental/InputsSnapshot';
+import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration';
 
 /**
  * Options for constructing a new Operation.
@@ -107,6 +111,8 @@ export class Operation {
    */
   public enabled: boolean;
 
+  private _stateHash: string | undefined;
+
   public constructor(options: IOperationOptions) {
     const { phase, project, runner, settings, logFilenameIdentifier } = options;
     this.associatedPhase = phase;
@@ -148,4 +154,83 @@ export class Operation {
     (this.dependencies as Set<Operation>).delete(dependency);
     (dependency.consumers as Set<Operation>).delete(this);
   }
+
+  public get stateHash(): string {
+    if (!this._stateHash) {
+      throw new Error(
+        'Operation state hash is not calculated yet, you must call `calculateStateHash` first.'
+      );
+    }
+    return this._stateHash;
+  }
+
+  public calculateStateHash(options: {
+    inputsSnapshot: IInputsSnapshot;
+    buildCacheConfiguration: BuildCacheConfiguration;
+  }): string {
+    if (this._stateHash) {
+      return this._stateHash;
+    }
+    const { inputsSnapshot, buildCacheConfiguration } = options;
+    const { cacheHashSalt } = buildCacheConfiguration;
+    // This redefinition is necessary due to limitations in TypeScript's control flow analysis, due to the nested closure.
+    const definitelyDefinedInputsSnapshot: IInputsSnapshot = inputsSnapshot;
+
+    // Build cache hashes are computed up front to ensure stability and to catch configuration errors early.
+    function getOrCreateOperationHash(operation: Operation): string {
+      if (operation._stateHash) {
+        return operation._stateHash;
+      }
+      // Examples of data in the config hash:
+      // - CLI parameters (ShellOperationRunner)
+      const configHash: string | undefined = operation.runner?.getConfigHash();
+
+      const { associatedProject, associatedPhase } = operation;
+      // Examples of data in the local state hash:
+      // - Environment variables specified in `dependsOnEnvVars`
+      // - Git hashes of tracked files in the associated project
+      // - Git hash of the shrinkwrap file for the project
+      // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project)
+      const localStateHash: string | undefined =
+        associatedProject &&
+        definitelyDefinedInputsSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name);
+
+      // The final state hashes of operation dependencies are factored into the hash to ensure that any
+      // state changes in dependencies will invalidate the cache.
+      const dependencyHashes: string[] = Array.from(operation.dependencies, getDependencyHash).sort();
+
+      const hasher: crypto.Hash = crypto.createHash('sha1');
+      // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content
+      // of the build cache.
+      hasher.update(`${RushConstants.buildCacheVersion}`);
+
+      if (cacheHashSalt !== undefined) {
+        // This allows repository owners to force a cache bust by changing the salt.
+        // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output.
+        hasher.update(cacheHashSalt);
+      }
+
+      for (const dependencyHash of dependencyHashes) {
+        hasher.update(dependencyHash);
+      }
+
+      if (localStateHash) {
+        hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`);
+      }
+
+      if (configHash) {
+        hasher.update(`${RushConstants.hashDelimiter}${configHash}`);
+      }
+
+      const hash: string = hasher.digest('hex');
+      operation._stateHash = hash;
+      return hash;
+    }
+
+    function getDependencyHash(operation: Operation): string {
+      return `${RushConstants.hashDelimiter}${operation.name}=${getOrCreateOperationHash(operation)}`;
+    }
+
+    return getOrCreateOperationHash(this);
+  }
 }