From 2eb0665e51f794b16e8306ea02da11e54970d3e1 Mon Sep 17 00:00:00 2001 From: Lincoln Date: Mon, 18 Nov 2024 08:13:51 +0000 Subject: [PATCH 1/8] Optimize the execution speed of Rush --- .../changes/@microsoft/rush/main_2024-11-18-08-13.json | 10 ++++++++++ .../package-deps-hash/main_2024-11-18-08-13.json | 10 ++++++++++ libraries/package-deps-hash/src/getRepoState.ts | 9 ++++++--- .../src/cli/scriptActions/PhasedScriptAction.ts | 3 ++- libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts | 7 +++++-- 5 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 common/changes/@microsoft/rush/main_2024-11-18-08-13.json create mode 100644 common/changes/@rushstack/package-deps-hash/main_2024-11-18-08-13.json diff --git a/common/changes/@microsoft/rush/main_2024-11-18-08-13.json b/common/changes/@microsoft/rush/main_2024-11-18-08-13.json new file mode 100644 index 00000000000..fc2320d2e1d --- /dev/null +++ b/common/changes/@microsoft/rush/main_2024-11-18-08-13.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Optimize the execution speed of Rush", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/changes/@rushstack/package-deps-hash/main_2024-11-18-08-13.json b/common/changes/@rushstack/package-deps-hash/main_2024-11-18-08-13.json new file mode 100644 index 00000000000..9f9751933bc --- /dev/null +++ b/common/changes/@rushstack/package-deps-hash/main_2024-11-18-08-13.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/package-deps-hash", + "comment": "Optimize the execution speed of Rush", + "type": "patch" + } + ], + "packageName": "@rushstack/package-deps-hash" +} \ No newline at end of file diff --git a/libraries/package-deps-hash/src/getRepoState.ts b/libraries/package-deps-hash/src/getRepoState.ts index c8992ac275d..ffa3d792399 100644 --- a/libraries/package-deps-hash/src/getRepoState.ts +++ b/libraries/package-deps-hash/src/getRepoState.ts @@ -364,7 +364,8 @@ export async function hashFilesAsync( export async function getRepoStateAsync( rootDirectory: string, additionalRelativePathsToHash?: string[], - gitPath?: string + gitPath?: string, + filterPath?: string[] ): Promise> { const statePromise: Promise = spawnGitAsync( gitPath, @@ -378,7 +379,8 @@ export async function getRepoStateAsync( '--full-name', // As of last commit 'HEAD', - '--' + '--', + ...(filterPath ? filterPath : []) ]), rootDirectory ).then(parseGitLsTree); @@ -396,7 +398,8 @@ export async function getRepoStateAsync( '--ignore-submodules', // Don't compare against the remote '--no-ahead-behind', - '--' + '--', + ...(filterPath ? filterPath : []) ]), rootDirectory ).then(parseGitStatus); diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 6eed33d9d00..3817faacc41 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -546,6 +546,7 @@ export class PhasedScriptAction extends BaseScriptAction { } = options; const { projectConfigurations } = initialCreateOperationsContext; + const { projectSelection } = initialCreateOperationsContext; const operations: Set = await this.hooks.createOperations.promise( new Set(), @@ -558,7 +559,7 @@ export class PhasedScriptAction extends BaseScriptAction { const analyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); const getInputsSnapshotAsync: GetInputsSnapshotAsyncFn | undefined = - await analyzer._tryGetSnapshotProviderAsync(projectConfigurations, terminal); + await analyzer._tryGetSnapshotProviderAsync(projectConfigurations, terminal, projectSelection); const initialSnapshot: IInputsSnapshot | undefined = await getInputsSnapshotAsync?.(); repoStateStopwatch.stop(); diff --git a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts index 942cbac6bdd..541edfb384d 100644 --- a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts +++ b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts @@ -201,7 +201,8 @@ export class ProjectChangeAnalyzer { */ public async _tryGetSnapshotProviderAsync( projectConfigurations: ReadonlyMap, - terminal: ITerminal + terminal: ITerminal, + projectSelection?: ReadonlySet ): Promise { try { const gitPath: string = this._git.getGitPathOrThrow(); @@ -295,10 +296,12 @@ export class ProjectChangeAnalyzer { const lookupByPath: IReadonlyLookupByPath = this._rushConfiguration.getProjectLookupForRoot(rootDirectory); + const filterPath: string[] = Array.from(projectSelection ?? []).map((project) => project.projectFolder); + return async function tryGetSnapshotAsync(): Promise { try { const [hashes, additionalFiles] = await Promise.all([ - getRepoStateAsync(rootDirectory, additionalRelativePathsToHash, gitPath), + getRepoStateAsync(rootDirectory, additionalRelativePathsToHash, gitPath, filterPath), getAdditionalFilesFromRushProjectConfigurationAsync( additionalGlobs, lookupByPath, From 1c4e8251b1cf2af9245aaf946f26b226499b8746 Mon Sep 17 00:00:00 2001 From: Lincoln Date: Mon, 18 Nov 2024 08:29:10 +0000 Subject: [PATCH 2/8] update documentation --- common/reviews/api/package-deps-hash.api.md | 2 +- common/reviews/api/rush-lib.api.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/reviews/api/package-deps-hash.api.md b/common/reviews/api/package-deps-hash.api.md index 1a7df0094e4..e6fd75084d2 100644 --- a/common/reviews/api/package-deps-hash.api.md +++ b/common/reviews/api/package-deps-hash.api.md @@ -20,7 +20,7 @@ export function getRepoChanges(currentWorkingDirectory: string, revision?: strin export function getRepoRoot(currentWorkingDirectory: string, gitPath?: string): string; // @beta -export function getRepoStateAsync(rootDirectory: string, additionalRelativePathsToHash?: string[], gitPath?: string): Promise>; +export function getRepoStateAsync(rootDirectory: string, additionalRelativePathsToHash?: string[], gitPath?: string, filterPath?: string[]): Promise>; // @beta export function hashFilesAsync(rootDirectory: string, filesToHash: Iterable | AsyncIterable, gitPath?: string): Promise>; diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 454ac3daa07..189c6ec4164 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -1129,7 +1129,7 @@ export class ProjectChangeAnalyzer { // (undocumented) protected getChangesByProject(lookup: LookupByPath, changedFiles: Map): Map>; // @internal - _tryGetSnapshotProviderAsync(projectConfigurations: ReadonlyMap, terminal: ITerminal): Promise; + _tryGetSnapshotProviderAsync(projectConfigurations: ReadonlyMap, terminal: ITerminal, projectSelection?: ReadonlySet): Promise; } // @public From 12c0c465fb0e45d517ddc85b6dc764948f9460dd Mon Sep 17 00:00:00 2001 From: Lincoln Date: Mon, 18 Nov 2024 23:01:11 +0000 Subject: [PATCH 3/8] Update documentation --- libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts index 94c31ea19c3..18d9b34a54a 100644 --- a/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts +++ b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts @@ -122,7 +122,9 @@ const { hashDelimiter } = RushConstants; */ export interface IInputsSnapshot { /** - * The raw hashes of all tracked files in the repository. + * The raw hashes of the files relevant to the projects we care about are stored. + * (e.g. when running `rush build`, the hashes of all tracked files in the repository are stored) + * (e.g. when running `rush build --only`, only the hashes of files under the specified project are stored) */ readonly hashes: ReadonlyMap; From 493e2aba3e8e6271c0666185d276b37240ca91c8 Mon Sep 17 00:00:00 2001 From: Lincoln Date: Sat, 23 Nov 2024 09:54:27 +0000 Subject: [PATCH 4/8] add enableSubpathScan to control if full scan the entire repository --- common/reviews/api/rush-lib.api.md | 1 + .../assets/rush-init/common/config/rush/experiments.json | 8 +++++++- libraries/rush-lib/src/api/ExperimentsConfiguration.ts | 6 ++++++ libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts | 9 ++++++++- .../rush-lib/src/logic/incremental/InputsSnapshot.ts | 4 +--- libraries/rush-lib/src/schemas/experiments.schema.json | 4 ++++ 6 files changed, 27 insertions(+), 5 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 189c6ec4164..cd12e7f6f39 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -481,6 +481,7 @@ export interface IExperimentsJson { buildCacheWithAllowWarningsInSuccessfulBuild?: boolean; buildSkipWithAllowWarningsInSuccessfulBuild?: boolean; cleanInstallAfterNpmrcChanges?: boolean; + enableSubpathScan?: boolean; forbidPhantomResolvableNodeModulesFolders?: boolean; generateProjectImpactGraphDuringRushUpdate?: boolean; noChmodFieldInTarHeaderNormalization?: boolean; diff --git a/libraries/rush-lib/assets/rush-init/common/config/rush/experiments.json b/libraries/rush-lib/assets/rush-init/common/config/rush/experiments.json index 8bc016fb1d2..0ed08b3edb9 100644 --- a/libraries/rush-lib/assets/rush-init/common/config/rush/experiments.json +++ b/libraries/rush-lib/assets/rush-init/common/config/rush/experiments.json @@ -102,5 +102,11 @@ /** * When using cobuilds, this experiment allows uncacheable operations to benefit from cobuild orchestration without using the build cache. */ - /*[LINE "HYPOTHETICAL"]*/ "allowCobuildWithoutCache": true + /*[LINE "HYPOTHETICAL"]*/ "allowCobuildWithoutCache": true, + + /** + * By default, rush perform a full scan of the entire repository. For example, Rush runs `git status` to check for local file changes. + * When this toggle is enabled, Rush will only scan specific paths, significantly speeding up Git operations. + */ + /*[LINE "HYPOTHETICAL"]*/ "enableSubpathScan": true } diff --git a/libraries/rush-lib/src/api/ExperimentsConfiguration.ts b/libraries/rush-lib/src/api/ExperimentsConfiguration.ts index e9a6f46bec0..6f29b79f5ca 100644 --- a/libraries/rush-lib/src/api/ExperimentsConfiguration.ts +++ b/libraries/rush-lib/src/api/ExperimentsConfiguration.ts @@ -113,6 +113,12 @@ export interface IExperimentsJson { * This is useful when you want to speed up operations that can't (or shouldn't) be cached. */ allowCobuildWithoutCache?: boolean; + + /** + * By default, rush perform a full scan of the entire repository. For example, Rush runs `git status` to check for local file changes. + * When this toggle is enabled, Rush will only scan specific paths, significantly speeding up Git operations. + */ + enableSubpathScan?: boolean; } const _EXPERIMENTS_JSON_SCHEMA: JsonSchema = JsonSchema.fromLoadedObject(schemaJson); diff --git a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts index 541edfb384d..eaf79c4c4e4 100644 --- a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts +++ b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts @@ -296,7 +296,14 @@ export class ProjectChangeAnalyzer { const lookupByPath: IReadonlyLookupByPath = this._rushConfiguration.getProjectLookupForRoot(rootDirectory); - const filterPath: string[] = Array.from(projectSelection ?? []).map((project) => project.projectFolder); + let filterPath: string[] = []; + + if ( + projectSelection && + this._rushConfiguration.experimentsConfiguration.configuration.enableSubpathScan + ) { + filterPath = Array.from(projectSelection).map(({ projectFolder }) => projectFolder); + } return async function tryGetSnapshotAsync(): Promise { try { diff --git a/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts index 18d9b34a54a..94c31ea19c3 100644 --- a/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts +++ b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts @@ -122,9 +122,7 @@ const { hashDelimiter } = RushConstants; */ export interface IInputsSnapshot { /** - * The raw hashes of the files relevant to the projects we care about are stored. - * (e.g. when running `rush build`, the hashes of all tracked files in the repository are stored) - * (e.g. when running `rush build --only`, only the hashes of files under the specified project are stored) + * The raw hashes of all tracked files in the repository. */ readonly hashes: ReadonlyMap; diff --git a/libraries/rush-lib/src/schemas/experiments.schema.json b/libraries/rush-lib/src/schemas/experiments.schema.json index d18ec334876..f75a021f36c 100644 --- a/libraries/rush-lib/src/schemas/experiments.schema.json +++ b/libraries/rush-lib/src/schemas/experiments.schema.json @@ -73,6 +73,10 @@ "rushAlerts": { "description": "(UNDER DEVELOPMENT) The Rush alerts feature provides a way to send announcements to engineers working in the monorepo, by printing directly in the user's shell window when they invoke Rush commands. This ensures that important notices will be seen by anyone doing active development, since people often ignore normal discussion group messages or don't know to subscribe.", "type": "boolean" + }, + "enableSubpathScan": { + "description": "By default, rush perform a full scan of the entire repository. For example, Rush runs `git status` to check for local file changes. When this toggle is enabled, Rush will only scan specific paths, significantly speeding up Git operations.", + "type": "boolean" } }, "additionalProperties": false From 75a0ad075858ad9a13286a385bc0262673eeeb95 Mon Sep 17 00:00:00 2001 From: Lincoln <778157949@qq.com> Date: Thu, 12 Dec 2024 06:35:50 +0800 Subject: [PATCH 5/8] Update common/changes/@rushstack/package-deps-hash/main_2024-11-18-08-13.json Co-authored-by: David Michon --- .../@rushstack/package-deps-hash/main_2024-11-18-08-13.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/changes/@rushstack/package-deps-hash/main_2024-11-18-08-13.json b/common/changes/@rushstack/package-deps-hash/main_2024-11-18-08-13.json index 9f9751933bc..83417c9734c 100644 --- a/common/changes/@rushstack/package-deps-hash/main_2024-11-18-08-13.json +++ b/common/changes/@rushstack/package-deps-hash/main_2024-11-18-08-13.json @@ -2,8 +2,8 @@ "changes": [ { "packageName": "@rushstack/package-deps-hash", - "comment": "Optimize the execution speed of Rush", - "type": "patch" + "comment": "Add a new optional parameter `filterPath` to `getRepoStateAsync` that limits the scope of the git query to only the specified subpaths. This can significantly improve the performance of the function when only part of the full repo data is necessary.", + "type": "minor" } ], "packageName": "@rushstack/package-deps-hash" From 72940ebfc8b3e1f696eb469c5c40024313f3cb3b Mon Sep 17 00:00:00 2001 From: Lincoln <778157949@qq.com> Date: Thu, 12 Dec 2024 06:36:59 +0800 Subject: [PATCH 6/8] Update libraries/package-deps-hash/src/getRepoState.ts Co-authored-by: David Michon --- libraries/package-deps-hash/src/getRepoState.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/package-deps-hash/src/getRepoState.ts b/libraries/package-deps-hash/src/getRepoState.ts index ffa3d792399..170ad543c2f 100644 --- a/libraries/package-deps-hash/src/getRepoState.ts +++ b/libraries/package-deps-hash/src/getRepoState.ts @@ -380,7 +380,7 @@ export async function getRepoStateAsync( // As of last commit 'HEAD', '--', - ...(filterPath ? filterPath : []) + ...(filterPath ?? []) ]), rootDirectory ).then(parseGitLsTree); From 6236ccdc66318a62c3f71ede74e7ca36507bea9c Mon Sep 17 00:00:00 2001 From: Lincoln <778157949@qq.com> Date: Thu, 12 Dec 2024 06:37:14 +0800 Subject: [PATCH 7/8] Update libraries/package-deps-hash/src/getRepoState.ts Co-authored-by: David Michon --- libraries/package-deps-hash/src/getRepoState.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/package-deps-hash/src/getRepoState.ts b/libraries/package-deps-hash/src/getRepoState.ts index 170ad543c2f..4b9bdcb57cc 100644 --- a/libraries/package-deps-hash/src/getRepoState.ts +++ b/libraries/package-deps-hash/src/getRepoState.ts @@ -399,7 +399,7 @@ export async function getRepoStateAsync( // Don't compare against the remote '--no-ahead-behind', '--', - ...(filterPath ? filterPath : []) + ...(filterPath ?? []) ]), rootDirectory ).then(parseGitStatus); From 4c094f8e15050781b324d43526199ef23cca61f8 Mon Sep 17 00:00:00 2001 From: Lincoln <778157949@qq.com> Date: Thu, 12 Dec 2024 06:39:51 +0800 Subject: [PATCH 8/8] Update common/changes/@microsoft/rush/main_2024-11-18-08-13.json Co-authored-by: David Michon --- common/changes/@microsoft/rush/main_2024-11-18-08-13.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/changes/@microsoft/rush/main_2024-11-18-08-13.json b/common/changes/@microsoft/rush/main_2024-11-18-08-13.json index fc2320d2e1d..5c272e51612 100644 --- a/common/changes/@microsoft/rush/main_2024-11-18-08-13.json +++ b/common/changes/@microsoft/rush/main_2024-11-18-08-13.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@microsoft/rush", - "comment": "Optimize the execution speed of Rush", + "comment": "Add a new experiment flag `enableSubpathScan` that, when invoking phased script commands with project selection parameters, such as `--to` or `--from`, only hashes files that are needed to compute the cache ids for the selected projects.", "type": "none" } ],