From b18dfab2d147c8404a3021b88adf8f14c1bf149f Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Thu, 26 Dec 2024 13:47:53 -0700 Subject: [PATCH 1/8] feat: improve access to operation build cache Signed-off-by: Aramis Sennyey --- common/reviews/api/rush-lib.api.md | 7 ++ .../src/logic/buildCache/ProjectBuildCache.ts | 41 ++++++--- .../operations/CacheableOperationPlugin.ts | 90 ++----------------- .../src/logic/operations/Operation.ts | 85 ++++++++++++++++++ 4 files changed, 131 insertions(+), 92 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index d739176d23a..bb60ada77f3 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -931,6 +931,11 @@ export class Operation { addDependency(dependency: Operation): void; readonly associatedPhase: IPhase | undefined; readonly associatedProject: RushConfigurationProject | undefined; + // (undocumented) + calculateStateHash(options: { + inputsSnapshot: IInputsSnapshot; + buildCacheConfiguration: BuildCacheConfiguration; + }): string; readonly consumers: ReadonlySet; deleteDependency(dependency: Operation): void; readonly dependencies: ReadonlySet; @@ -940,6 +945,8 @@ export class Operation { get name(): string | undefined; runner: IOperationRunner | undefined; settings: IOperationSettings | undefined; + // (undocumented) + get stateHash(): string; weight: number; } diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index cbba6950d83..5b1e6fb00f7 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -13,33 +13,37 @@ import type { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider'; import type { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider'; import { TarExecutable } from '../../utilities/TarExecutable'; import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration'; +import type { Operation } from '../operations/Operation'; -export interface IProjectBuildCacheOptions { +export interface IOperationBuildCacheOptions { /** * The repo-wide configuration for the build cache. */ buildCacheConfiguration: BuildCacheConfiguration; - /** - * The project to be cached. - */ - project: RushConfigurationProject; /** * Value from rush-project.json */ projectOutputFolderNames: ReadonlyArray; - /** - * The hash of all relevant inputs and configuration that uniquely identifies this execution. - */ - operationStateHash: string; /** * The terminal to use for logging. */ terminal: ITerminal; +} + +export type IProjectBuildCacheOptions = IOperationBuildCacheOptions & { + /** + * The project to be cached. + */ + project: RushConfigurationProject; + /** + * The hash of all relevant inputs and configuration that uniquely identifies this execution. + */ + operationStateHash: string; /** * The name of the phase that is being cached. */ phaseName: string; -} +}; interface IPathsToCache { filteredOutputFolderNames: string[]; @@ -94,6 +98,23 @@ export class ProjectBuildCache { return new ProjectBuildCache(cacheId, options); } + public static forOperation(operation: Operation, options: IOperationBuildCacheOptions): ProjectBuildCache { + if (!operation.associatedProject) { + throw new InternalError('Operation must have an associated project'); + } + if (!operation.associatedPhase) { + throw new InternalError('Operation must have an associated phase'); + } + const buildCacheOptions: IProjectBuildCacheOptions = { + ...options, + project: operation.associatedProject, + phaseName: operation.associatedPhase.name, + operationStateHash: operation.stateHash + }; + const cacheId: string | undefined = ProjectBuildCache._getCacheId(buildCacheOptions); + return new ProjectBuildCache(cacheId, buildCacheOptions); + } + public async tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise { const cacheId: string | undefined = specifiedCacheId || this._cacheId; if (!cacheId) { diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 5632236f735..836de2d700d 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -35,7 +35,6 @@ import type { IPhase } from '../../api/CommandLineConfiguration'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; import type { OperationExecutionRecord } from './OperationExecutionRecord'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; const PERIODIC_CALLBACK_INTERVAL_IN_SECONDS: number = 10; @@ -88,8 +87,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { const { allowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration } = this._options; - const { cacheHashSalt } = buildCacheConfiguration; - hooks.beforeExecuteOperations.tap( PLUGIN_NAME, ( @@ -104,76 +101,15 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { ); } - // This redefinition is necessary due to limitations in TypeScript's control flow analysis, due to the nested closure. - const definitelyDefinedInputsSnapshot: IInputsSnapshot = inputsSnapshot; - const disjointSet: DisjointSet | undefined = cobuildConfiguration?.cobuildFeatureEnabled ? new DisjointSet() : undefined; - const hashByOperation: Map = new Map(); - // Build cache hashes are computed up front to ensure stability and to catch configuration errors early. - function getOrCreateOperationHash(operation: Operation): string { - const cachedHash: string | undefined = hashByOperation.get(operation); - if (cachedHash !== undefined) { - return cachedHash; - } - - // Examples of data in the config hash: - // - CLI parameters (ShellOperationRunner) - const configHash: string | undefined = operation.runner?.getConfigHash(); - - const { associatedProject, associatedPhase } = operation; - // Examples of data in the local state hash: - // - Environment variables specified in `dependsOnEnvVars` - // - Git hashes of tracked files in the associated project - // - Git hash of the shrinkwrap file for the project - // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) - const localStateHash: string | undefined = - associatedProject && - definitelyDefinedInputsSnapshot.getOperationOwnStateHash( - associatedProject, - associatedPhase?.name - ); - - // The final state hashes of operation dependencies are factored into the hash to ensure that any - // state changes in dependencies will invalidate the cache. - const dependencyHashes: string[] = Array.from(operation.dependencies, getDependencyHash).sort(); - - const hasher: crypto.Hash = crypto.createHash('sha1'); - // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content - // of the build cache. - hasher.update(`${RushConstants.buildCacheVersion}`); - - if (cacheHashSalt !== undefined) { - // This allows repository owners to force a cache bust by changing the salt. - // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. - hasher.update(cacheHashSalt); - } - - for (const dependencyHash of dependencyHashes) { - hasher.update(dependencyHash); - } - - if (localStateHash) { - hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`); - } - - if (configHash) { - hasher.update(`${RushConstants.hashDelimiter}${configHash}`); - } - - const hashString: string = hasher.digest('hex'); - - hashByOperation.set(operation, hashString); - return hashString; - } - - function getDependencyHash(operation: Operation): string { - return `${RushConstants.hashDelimiter}${operation.name}=${getOrCreateOperationHash(operation)}`; - } - for (const [operation, record] of recordByOperation) { + const stateHash: string = operation.calculateStateHash({ + inputsSnapshot, + buildCacheConfiguration + }); const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation; if (!associatedProject || !associatedPhase || !runner) { return; @@ -188,7 +124,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // depending on the selected phase. const fileHashes: ReadonlyMap | undefined = inputsSnapshot.getTrackedFileHashesForOperation(associatedProject, phaseName); - const stateHash: string = getOrCreateOperationHash(operation); const cacheDisabledReason: string | undefined = projectConfiguration ? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName, operation.isNoOp) @@ -325,10 +260,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { let projectBuildCache: ProjectBuildCache | undefined = this._tryGetProjectBuildCache({ buildCacheContext, buildCacheConfiguration, - rushProject: project, - phase, terminal: buildCacheTerminal, - operation: operation + operation }); // Try to acquire the cobuild lock @@ -639,15 +572,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { private _tryGetProjectBuildCache({ buildCacheConfiguration, buildCacheContext, - rushProject, - phase, terminal, operation }: { buildCacheContext: IOperationBuildCacheContext; buildCacheConfiguration: BuildCacheConfiguration | undefined; - rushProject: RushConfigurationProject; - phase: IPhase; terminal: ITerminal; operation: Operation; }): ProjectBuildCache | undefined { @@ -658,20 +587,17 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return; } - const { outputFolderNames, stateHash: operationStateHash } = buildCacheContext; + const { outputFolderNames } = buildCacheContext; if (!outputFolderNames || !buildCacheConfiguration) { // Unreachable, since this will have set `cacheDisabledReason`. return; } // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent - buildCacheContext.operationBuildCache = ProjectBuildCache.getProjectBuildCache({ - project: rushProject, + buildCacheContext.operationBuildCache = ProjectBuildCache.forOperation(operation, { projectOutputFolderNames: outputFolderNames, buildCacheConfiguration, - terminal, - operationStateHash, - phaseName: phase.name + terminal }); } diff --git a/libraries/rush-lib/src/logic/operations/Operation.ts b/libraries/rush-lib/src/logic/operations/Operation.ts index bbc8d5d2af8..b496950d729 100644 --- a/libraries/rush-lib/src/logic/operations/Operation.ts +++ b/libraries/rush-lib/src/logic/operations/Operation.ts @@ -5,6 +5,10 @@ import type { RushConfigurationProject } from '../../api/RushConfigurationProjec import type { IPhase } from '../../api/CommandLineConfiguration'; import type { IOperationRunner } from './IOperationRunner'; import type { IOperationSettings } from '../../api/RushProjectConfiguration'; +import { RushConstants } from '../RushConstants'; +import * as crypto from 'crypto'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; /** * Options for constructing a new Operation. @@ -107,6 +111,8 @@ export class Operation { */ public enabled: boolean; + private _stateHash: string | undefined; + public constructor(options: IOperationOptions) { const { phase, project, runner, settings, logFilenameIdentifier } = options; this.associatedPhase = phase; @@ -148,4 +154,83 @@ export class Operation { (this.dependencies as Set).delete(dependency); (dependency.consumers as Set).delete(this); } + + public get stateHash(): string { + if (!this._stateHash) { + throw new Error( + 'Operation state hash is not calculated yet, you must call `calculateStateHash` first.' + ); + } + return this._stateHash; + } + + public calculateStateHash(options: { + inputsSnapshot: IInputsSnapshot; + buildCacheConfiguration: BuildCacheConfiguration; + }): string { + if (this._stateHash) { + return this._stateHash; + } + const { inputsSnapshot, buildCacheConfiguration } = options; + const { cacheHashSalt } = buildCacheConfiguration; + // This redefinition is necessary due to limitations in TypeScript's control flow analysis, due to the nested closure. + const definitelyDefinedInputsSnapshot: IInputsSnapshot = inputsSnapshot; + + // Build cache hashes are computed up front to ensure stability and to catch configuration errors early. + function getOrCreateOperationHash(operation: Operation): string { + if (operation._stateHash) { + return operation._stateHash; + } + // Examples of data in the config hash: + // - CLI parameters (ShellOperationRunner) + const configHash: string | undefined = operation.runner?.getConfigHash(); + + const { associatedProject, associatedPhase } = operation; + // Examples of data in the local state hash: + // - Environment variables specified in `dependsOnEnvVars` + // - Git hashes of tracked files in the associated project + // - Git hash of the shrinkwrap file for the project + // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) + const localStateHash: string | undefined = + associatedProject && + definitelyDefinedInputsSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name); + + // The final state hashes of operation dependencies are factored into the hash to ensure that any + // state changes in dependencies will invalidate the cache. + const dependencyHashes: string[] = Array.from(operation.dependencies, getDependencyHash).sort(); + + const hasher: crypto.Hash = crypto.createHash('sha1'); + // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content + // of the build cache. + hasher.update(`${RushConstants.buildCacheVersion}`); + + if (cacheHashSalt !== undefined) { + // This allows repository owners to force a cache bust by changing the salt. + // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. + hasher.update(cacheHashSalt); + } + + for (const dependencyHash of dependencyHashes) { + hasher.update(dependencyHash); + } + + if (localStateHash) { + hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`); + } + + if (configHash) { + hasher.update(`${RushConstants.hashDelimiter}${configHash}`); + } + + const hash: string = hasher.digest('hex'); + operation._stateHash = hash; + return hash; + } + + function getDependencyHash(operation: Operation): string { + return `${RushConstants.hashDelimiter}${operation.name}=${getOrCreateOperationHash(operation)}`; + } + + return getOrCreateOperationHash(this); + } } From 944f86c22f8f20c450c643b6222e539d70e8c52d Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Thu, 26 Dec 2024 16:12:46 -0700 Subject: [PATCH 2/8] add changelog Signed-off-by: Aramis Sennyey --- .../rush/sennyeya-cache-id_2024-12-26-23-12.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 common/changes/@microsoft/rush/sennyeya-cache-id_2024-12-26-23-12.json diff --git a/common/changes/@microsoft/rush/sennyeya-cache-id_2024-12-26-23-12.json b/common/changes/@microsoft/rush/sennyeya-cache-id_2024-12-26-23-12.json new file mode 100644 index 00000000000..136490cc78e --- /dev/null +++ b/common/changes/@microsoft/rush/sennyeya-cache-id_2024-12-26-23-12.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "comment": "Simplifies the process of going from operation to build cache ID.", + "type": "none", + "packageName": "@microsoft/rush" + } + ], + "packageName": "@microsoft/rush", + "email": "aramissennyeydd@users.noreply.github.com" +} \ No newline at end of file From eb6010b95a649b49d3c464d7df4e6677b440edd2 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Fri, 3 Jan 2025 10:17:16 -0700 Subject: [PATCH 3/8] move state hash calculation to OperationExecutionRecord Signed-off-by: Aramis Sennyey --- common/reviews/api/rush-lib.api.md | 7 -- .../src/logic/buildCache/ProjectBuildCache.ts | 20 +++-- .../operations/CacheableOperationPlugin.ts | 25 +++--- .../src/logic/operations/Operation.ts | 83 ------------------- .../operations/OperationExecutionRecord.ts | 70 ++++++++++++++++ 5 files changed, 97 insertions(+), 108 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index bb60ada77f3..d739176d23a 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -931,11 +931,6 @@ export class Operation { addDependency(dependency: Operation): void; readonly associatedPhase: IPhase | undefined; readonly associatedProject: RushConfigurationProject | undefined; - // (undocumented) - calculateStateHash(options: { - inputsSnapshot: IInputsSnapshot; - buildCacheConfiguration: BuildCacheConfiguration; - }): string; readonly consumers: ReadonlySet; deleteDependency(dependency: Operation): void; readonly dependencies: ReadonlySet; @@ -945,8 +940,6 @@ export class Operation { get name(): string | undefined; runner: IOperationRunner | undefined; settings: IOperationSettings | undefined; - // (undocumented) - get stateHash(): string; weight: number; } diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index 5b1e6fb00f7..9b3e495c70d 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -13,17 +13,13 @@ import type { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider'; import type { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider'; import { TarExecutable } from '../../utilities/TarExecutable'; import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration'; -import type { Operation } from '../operations/Operation'; +import type { OperationExecutionRecord } from '../operations/OperationExecutionRecord'; export interface IOperationBuildCacheOptions { /** * The repo-wide configuration for the build cache. */ buildCacheConfiguration: BuildCacheConfiguration; - /** - * Value from rush-project.json - */ - projectOutputFolderNames: ReadonlyArray; /** * The terminal to use for logging. */ @@ -31,6 +27,10 @@ export interface IOperationBuildCacheOptions { } export type IProjectBuildCacheOptions = IOperationBuildCacheOptions & { + /** + * Value from rush-project.json + */ + projectOutputFolderNames: ReadonlyArray; /** * The project to be cached. */ @@ -98,17 +98,25 @@ export class ProjectBuildCache { return new ProjectBuildCache(cacheId, options); } - public static forOperation(operation: Operation, options: IOperationBuildCacheOptions): ProjectBuildCache { + public static forOperation( + operation: OperationExecutionRecord, + options: IOperationBuildCacheOptions + ): ProjectBuildCache { if (!operation.associatedProject) { throw new InternalError('Operation must have an associated project'); } if (!operation.associatedPhase) { throw new InternalError('Operation must have an associated phase'); } + const outputFolders: string[] = [...(operation.operation.settings?.outputFolderNames ?? [])]; + if (operation.metadataFolderPath) { + outputFolders.push(operation.metadataFolderPath); + } const buildCacheOptions: IProjectBuildCacheOptions = { ...options, project: operation.associatedProject, phaseName: operation.associatedPhase.name, + projectOutputFolderNames: outputFolders, operationStateHash: operation.stateHash }; const cacheId: string | undefined = ProjectBuildCache._getCacheId(buildCacheOptions); diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 836de2d700d..c8d4d4f8467 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -106,7 +106,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { : undefined; for (const [operation, record] of recordByOperation) { - const stateHash: string = operation.calculateStateHash({ + const stateHash: string = (record as OperationExecutionRecord).calculateStateHash({ inputsSnapshot, buildCacheConfiguration }); @@ -261,7 +261,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheContext, buildCacheConfiguration, terminal: buildCacheTerminal, - operation + record }); // Try to acquire the cobuild lock @@ -573,32 +573,33 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheConfiguration, buildCacheContext, terminal, - operation + record }: { buildCacheContext: IOperationBuildCacheContext; buildCacheConfiguration: BuildCacheConfiguration | undefined; terminal: ITerminal; - operation: Operation; + record: OperationExecutionRecord; }): ProjectBuildCache | undefined { if (!buildCacheContext.operationBuildCache) { const { cacheDisabledReason } = buildCacheContext; - if (cacheDisabledReason && !operation.settings?.allowCobuildWithoutCache) { + if (cacheDisabledReason && !record.operation.settings?.allowCobuildWithoutCache) { terminal.writeVerboseLine(cacheDisabledReason); return; } - const { outputFolderNames } = buildCacheContext; - if (!outputFolderNames || !buildCacheConfiguration) { + if (!buildCacheConfiguration) { // Unreachable, since this will have set `cacheDisabledReason`. return; } // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent - buildCacheContext.operationBuildCache = ProjectBuildCache.forOperation(operation, { - projectOutputFolderNames: outputFolderNames, - buildCacheConfiguration, - terminal - }); + buildCacheContext.operationBuildCache = ProjectBuildCache.forOperation( + record as OperationExecutionRecord, + { + buildCacheConfiguration, + terminal + } + ); } return buildCacheContext.operationBuildCache; diff --git a/libraries/rush-lib/src/logic/operations/Operation.ts b/libraries/rush-lib/src/logic/operations/Operation.ts index b496950d729..93929100f7c 100644 --- a/libraries/rush-lib/src/logic/operations/Operation.ts +++ b/libraries/rush-lib/src/logic/operations/Operation.ts @@ -5,10 +5,6 @@ import type { RushConfigurationProject } from '../../api/RushConfigurationProjec import type { IPhase } from '../../api/CommandLineConfiguration'; import type { IOperationRunner } from './IOperationRunner'; import type { IOperationSettings } from '../../api/RushProjectConfiguration'; -import { RushConstants } from '../RushConstants'; -import * as crypto from 'crypto'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; -import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; /** * Options for constructing a new Operation. @@ -154,83 +150,4 @@ export class Operation { (this.dependencies as Set).delete(dependency); (dependency.consumers as Set).delete(this); } - - public get stateHash(): string { - if (!this._stateHash) { - throw new Error( - 'Operation state hash is not calculated yet, you must call `calculateStateHash` first.' - ); - } - return this._stateHash; - } - - public calculateStateHash(options: { - inputsSnapshot: IInputsSnapshot; - buildCacheConfiguration: BuildCacheConfiguration; - }): string { - if (this._stateHash) { - return this._stateHash; - } - const { inputsSnapshot, buildCacheConfiguration } = options; - const { cacheHashSalt } = buildCacheConfiguration; - // This redefinition is necessary due to limitations in TypeScript's control flow analysis, due to the nested closure. - const definitelyDefinedInputsSnapshot: IInputsSnapshot = inputsSnapshot; - - // Build cache hashes are computed up front to ensure stability and to catch configuration errors early. - function getOrCreateOperationHash(operation: Operation): string { - if (operation._stateHash) { - return operation._stateHash; - } - // Examples of data in the config hash: - // - CLI parameters (ShellOperationRunner) - const configHash: string | undefined = operation.runner?.getConfigHash(); - - const { associatedProject, associatedPhase } = operation; - // Examples of data in the local state hash: - // - Environment variables specified in `dependsOnEnvVars` - // - Git hashes of tracked files in the associated project - // - Git hash of the shrinkwrap file for the project - // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) - const localStateHash: string | undefined = - associatedProject && - definitelyDefinedInputsSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name); - - // The final state hashes of operation dependencies are factored into the hash to ensure that any - // state changes in dependencies will invalidate the cache. - const dependencyHashes: string[] = Array.from(operation.dependencies, getDependencyHash).sort(); - - const hasher: crypto.Hash = crypto.createHash('sha1'); - // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content - // of the build cache. - hasher.update(`${RushConstants.buildCacheVersion}`); - - if (cacheHashSalt !== undefined) { - // This allows repository owners to force a cache bust by changing the salt. - // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. - hasher.update(cacheHashSalt); - } - - for (const dependencyHash of dependencyHashes) { - hasher.update(dependencyHash); - } - - if (localStateHash) { - hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`); - } - - if (configHash) { - hasher.update(`${RushConstants.hashDelimiter}${configHash}`); - } - - const hash: string = hasher.digest('hex'); - operation._stateHash = hash; - return hash; - } - - function getDependencyHash(operation: Operation): string { - return `${RushConstants.hashDelimiter}${operation.name}=${getOrCreateOperationHash(operation)}`; - } - - return getOrCreateOperationHash(this); - } } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index d9c4a2b82a9..21236ae51dc 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -1,5 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import * as crypto from 'crypto'; import { type ITerminal, @@ -29,6 +30,9 @@ import { initializeProjectLogFilesAsync } from './ProjectLogWritable'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; +import { RushConstants } from '../RushConstants'; export interface IOperationExecutionRecordContext { streamCollator: StreamCollator; @@ -114,6 +118,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera private _collatedWriter: CollatedWriter | undefined = undefined; private _status: OperationStatus; + private _stateHash: string | undefined; public constructor(operation: Operation, context: IOperationExecutionRecordContext) { const { runner, associatedPhase, associatedProject } = operation; @@ -206,6 +211,15 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera return !this.operation.enabled || this.runner.silent; } + public get stateHash(): string { + if (!this._stateHash) { + throw new Error( + 'Operation state hash is not calculated yet, you must call `calculateStateHash` first.' + ); + } + return this._stateHash; + } + /** * {@inheritdoc IOperationRunnerContext.runWithTerminalAsync} */ @@ -335,4 +349,60 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera } } } + + public calculateStateHash(options: { + inputsSnapshot: IInputsSnapshot; + buildCacheConfiguration: BuildCacheConfiguration; + }): string { + if (this._stateHash) { + return this._stateHash; + } + const { inputsSnapshot, buildCacheConfiguration } = options; + const { cacheHashSalt } = buildCacheConfiguration; + + // Examples of data in the config hash: + // - CLI parameters (ShellOperationRunner) + const configHash: string = this.runner.getConfigHash(); + + const { associatedProject, associatedPhase } = this; + // Examples of data in the local state hash: + // - Environment variables specified in `dependsOnEnvVars` + // - Git hashes of tracked files in the associated project + // - Git hash of the shrinkwrap file for the project + // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) + const localStateHash: string | undefined = + associatedProject && inputsSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name); + + // The final state hashes of operation dependencies are factored into the hash to ensure that any + // state changes in dependencies will invalidate the cache. + const dependencyHashes: string[] = Array.from(this.dependencies, (record) => { + return `${RushConstants.hashDelimiter}${record.name}=${record.calculateStateHash(options)}`; + }).sort(); + + const hasher: crypto.Hash = crypto.createHash('sha1'); + // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content + // of the build cache. + hasher.update(`${RushConstants.buildCacheVersion}`); + + if (cacheHashSalt !== undefined) { + // This allows repository owners to force a cache bust by changing the salt. + // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. + hasher.update(cacheHashSalt); + } + + for (const dependencyHash of dependencyHashes) { + hasher.update(dependencyHash); + } + + if (localStateHash) { + hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`); + } + + hasher.update(`${RushConstants.hashDelimiter}${configHash}`); + + const hash: string = hasher.digest('hex'); + this._stateHash = hash; + + return this.stateHash; + } } From 8ec37cc3a3947ca97b7e8130a5e09cf55c17e1b6 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Fri, 3 Jan 2025 18:02:04 -0700 Subject: [PATCH 4/8] address PR feedback Signed-off-by: Aramis Sennyey --- .../operations/CacheableOperationPlugin.ts | 23 +++-- .../operations/OperationExecutionRecord.ts | 86 +++++++++---------- 2 files changed, 54 insertions(+), 55 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index c8d4d4f8467..b83703d2fec 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -115,6 +115,16 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return; } + const { cacheHashSalt } = buildCacheConfiguration; + const hasher: crypto.Hash = crypto.createHash('sha1'); + hasher.update(stateHash); + if (cacheHashSalt !== undefined) { + // This allows repository owners to force a cache bust by changing the salt. + // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. + hasher.update(cacheHashSalt); + } + const finalStateHash: string = hasher.digest('hex'); + const { name: phaseName } = associatedPhase; const projectConfiguration: RushProjectConfiguration | undefined = @@ -149,7 +159,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { isCacheReadAllowed: isIncrementalBuildAllowed, operationBuildCache: undefined, outputFolderNames, - stateHash, + stateHash: finalStateHash, cacheDisabledReason, cobuildLock: undefined, cobuildClusterId: undefined, @@ -593,13 +603,10 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent - buildCacheContext.operationBuildCache = ProjectBuildCache.forOperation( - record as OperationExecutionRecord, - { - buildCacheConfiguration, - terminal - } - ); + buildCacheContext.operationBuildCache = ProjectBuildCache.forOperation(record, { + buildCacheConfiguration, + terminal + }); } return buildCacheContext.operationBuildCache; diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 21236ae51dc..a730e7d7be4 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -354,55 +354,47 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera inputsSnapshot: IInputsSnapshot; buildCacheConfiguration: BuildCacheConfiguration; }): string { - if (this._stateHash) { - return this._stateHash; - } - const { inputsSnapshot, buildCacheConfiguration } = options; - const { cacheHashSalt } = buildCacheConfiguration; - - // Examples of data in the config hash: - // - CLI parameters (ShellOperationRunner) - const configHash: string = this.runner.getConfigHash(); - - const { associatedProject, associatedPhase } = this; - // Examples of data in the local state hash: - // - Environment variables specified in `dependsOnEnvVars` - // - Git hashes of tracked files in the associated project - // - Git hash of the shrinkwrap file for the project - // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) - const localStateHash: string | undefined = - associatedProject && inputsSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name); - - // The final state hashes of operation dependencies are factored into the hash to ensure that any - // state changes in dependencies will invalidate the cache. - const dependencyHashes: string[] = Array.from(this.dependencies, (record) => { - return `${RushConstants.hashDelimiter}${record.name}=${record.calculateStateHash(options)}`; - }).sort(); - - const hasher: crypto.Hash = crypto.createHash('sha1'); - // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content - // of the build cache. - hasher.update(`${RushConstants.buildCacheVersion}`); - - if (cacheHashSalt !== undefined) { - // This allows repository owners to force a cache bust by changing the salt. - // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. - hasher.update(cacheHashSalt); - } - - for (const dependencyHash of dependencyHashes) { - hasher.update(dependencyHash); - } - - if (localStateHash) { - hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`); - } + if (!this._stateHash) { + const { inputsSnapshot } = options; + + // Examples of data in the config hash: + // - CLI parameters (ShellOperationRunner) + const configHash: string = this.runner.getConfigHash(); + + const { associatedProject, associatedPhase } = this; + // Examples of data in the local state hash: + // - Environment variables specified in `dependsOnEnvVars` + // - Git hashes of tracked files in the associated project + // - Git hash of the shrinkwrap file for the project + // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) + const localStateHash: string | undefined = + associatedProject && + inputsSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name); + + // The final state hashes of operation dependencies are factored into the hash to ensure that any + // state changes in dependencies will invalidate the cache. + const dependencyHashes: string[] = Array.from(this.dependencies, (record) => { + return `${RushConstants.hashDelimiter}${record.name}=${record.calculateStateHash(options)}`; + }).sort(); + + const hasher: crypto.Hash = crypto.createHash('sha1'); + // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content + // of the build cache. + hasher.update(`${RushConstants.buildCacheVersion}`); + + for (const dependencyHash of dependencyHashes) { + hasher.update(dependencyHash); + } - hasher.update(`${RushConstants.hashDelimiter}${configHash}`); + if (localStateHash) { + hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`); + } - const hash: string = hasher.digest('hex'); - this._stateHash = hash; + hasher.update(`${RushConstants.hashDelimiter}${configHash}`); - return this.stateHash; + const hash: string = hasher.digest('hex'); + this._stateHash = hash; + } + return this._stateHash; } } From 7795f0f3987440a8d5a71c5a7fc2cd025d3eb689 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Fri, 3 Jan 2025 18:03:28 -0700 Subject: [PATCH 5/8] remove unused property Signed-off-by: Aramis Sennyey --- .../src/logic/operations/CacheableOperationPlugin.ts | 3 +-- .../src/logic/operations/OperationExecutionRecord.ts | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index b83703d2fec..b8e7f6fca4a 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -107,8 +107,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { for (const [operation, record] of recordByOperation) { const stateHash: string = (record as OperationExecutionRecord).calculateStateHash({ - inputsSnapshot, - buildCacheConfiguration + inputsSnapshot }); const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation; if (!associatedProject || !associatedPhase || !runner) { diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index a730e7d7be4..4543149fb43 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -350,10 +350,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera } } - public calculateStateHash(options: { - inputsSnapshot: IInputsSnapshot; - buildCacheConfiguration: BuildCacheConfiguration; - }): string { + public calculateStateHash(options: { inputsSnapshot: IInputsSnapshot }): string { if (!this._stateHash) { const { inputsSnapshot } = options; From f5f4b74874a08ffeae3f53d0489282f2434a4086 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Fri, 3 Jan 2025 18:14:44 -0700 Subject: [PATCH 6/8] remove unused import Signed-off-by: Aramis Sennyey --- .../rush-lib/src/logic/operations/OperationExecutionRecord.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 4543149fb43..64b8b9a63c4 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -31,7 +31,6 @@ import { } from './ProjectLogWritable'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; -import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import { RushConstants } from '../RushConstants'; export interface IOperationExecutionRecordContext { From 28dcf9239925046c114b8474714d4792a71ec283 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey <159921952+aramissennyeydd@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:14:25 -0500 Subject: [PATCH 7/8] Update libraries/rush-lib/src/logic/operations/Operation.ts Co-authored-by: David Michon --- libraries/rush-lib/src/logic/operations/Operation.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/Operation.ts b/libraries/rush-lib/src/logic/operations/Operation.ts index 93929100f7c..bbc8d5d2af8 100644 --- a/libraries/rush-lib/src/logic/operations/Operation.ts +++ b/libraries/rush-lib/src/logic/operations/Operation.ts @@ -107,8 +107,6 @@ export class Operation { */ public enabled: boolean; - private _stateHash: string | undefined; - public constructor(options: IOperationOptions) { const { phase, project, runner, settings, logFilenameIdentifier } = options; this.associatedPhase = phase; From 7977f26728e6d0ade9cdc4773861d0835f71df87 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Wed, 8 Jan 2025 16:17:57 -0500 Subject: [PATCH 8/8] address pr feedback Signed-off-by: Aramis Sennyey --- .../operations/CacheableOperationPlugin.ts | 15 +++------------ .../operations/OperationExecutionRecord.ts | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index b8e7f6fca4a..37ba2d626e7 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -107,23 +107,14 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { for (const [operation, record] of recordByOperation) { const stateHash: string = (record as OperationExecutionRecord).calculateStateHash({ - inputsSnapshot + inputsSnapshot, + buildCacheConfiguration }); const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation; if (!associatedProject || !associatedPhase || !runner) { return; } - const { cacheHashSalt } = buildCacheConfiguration; - const hasher: crypto.Hash = crypto.createHash('sha1'); - hasher.update(stateHash); - if (cacheHashSalt !== undefined) { - // This allows repository owners to force a cache bust by changing the salt. - // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. - hasher.update(cacheHashSalt); - } - const finalStateHash: string = hasher.digest('hex'); - const { name: phaseName } = associatedPhase; const projectConfiguration: RushProjectConfiguration | undefined = @@ -158,7 +149,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { isCacheReadAllowed: isIncrementalBuildAllowed, operationBuildCache: undefined, outputFolderNames, - stateHash: finalStateHash, + stateHash, cacheDisabledReason, cobuildLock: undefined, cobuildClusterId: undefined, diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 64b8b9a63c4..54ca0331531 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -32,6 +32,7 @@ import { import type { IOperationExecutionResult } from './IOperationExecutionResult'; import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; import { RushConstants } from '../RushConstants'; +import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; export interface IOperationExecutionRecordContext { streamCollator: StreamCollator; @@ -349,9 +350,15 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera } } - public calculateStateHash(options: { inputsSnapshot: IInputsSnapshot }): string { + public calculateStateHash(options: { + inputsSnapshot: IInputsSnapshot; + buildCacheConfiguration: BuildCacheConfiguration; + }): string { if (!this._stateHash) { - const { inputsSnapshot } = options; + const { + inputsSnapshot, + buildCacheConfiguration: { cacheHashSalt } + } = options; // Examples of data in the config hash: // - CLI parameters (ShellOperationRunner) @@ -378,6 +385,12 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera // of the build cache. hasher.update(`${RushConstants.buildCacheVersion}`); + if (cacheHashSalt !== undefined) { + // This allows repository owners to force a cache bust by changing the salt. + // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. + hasher.update(cacheHashSalt); + } + for (const dependencyHash of dependencyHashes) { hasher.update(dependencyHash); }