Skip to content

[rush] move sharded operation config to its own block #4923

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
"outputFolderNames": ["dist"],
"sharding": {
"count": 4,
"outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}",

"shardOperationSettings": {
"weight": 4
}
"outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}"
}
},
{
"operationName": "_phase:build:shard",
"weight": 4
}
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
"outputFolderNames": ["dist"],
"sharding": {
"count": 5,
"outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}",
"shardOperationSettings": {
"weight": 10
}
"outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}"
}
},
{
"operationName": "_phase:build:shard",
"weight": 10
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"changes": [
{
"comment": "Remove sharding.shardOperationSettings. Use a new operationSettings object with a name of ':shard'",
"type": "none",
"packageName": "@microsoft/rush"
}
],
"packageName": "@microsoft/rush",
"email": "aramissennyeydd@users.noreply.github.com"
}
3 changes: 0 additions & 3 deletions common/reviews/api/rush-lib.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -785,9 +785,6 @@ export interface IRushPhaseSharding {
count: number;
outputFolderArgumentFormat?: string;
shardArgumentFormat?: string;
shardOperationSettings?: {
weight?: number;
};
}

// @beta (undocumented)
Expand Down
11 changes: 0 additions & 11 deletions libraries/rush-lib/src/api/RushProjectConfiguration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,17 +63,6 @@ export interface IRushPhaseSharding {
* @defaultValue `--shard-output-folder=.rush/operations/{phaseName}/shards/{shardIndex}`.
*/
outputFolderArgumentFormat?: string;

/**
* Configuration for the shard operation. All other configuration applies to the collator operation.
*/
shardOperationSettings?: {
/**
* How many concurrency units this operation should take up during execution. The maximum concurrent units is
* determined by the -p flag.
*/
weight?: number;
};
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// See LICENSE in the project root for license information.

import type { IPhase } from '../../api/CommandLineConfiguration';
import type { RushProjectConfiguration } from '../../api/RushProjectConfiguration';
import type {
ICreateOperationsContext,
IPhasedCommandPlugin,
Expand Down Expand Up @@ -45,7 +46,7 @@ export class ShardedPhasedOperationPlugin implements IPhasedCommandPlugin {
}

function spliceShards(existingOperations: Set<Operation>, context: ICreateOperationsContext): Set<Operation> {
const { rushConfiguration } = context;
const { rushConfiguration, projectConfigurations } = context;

const getCustomParameterValuesForPhase: (phase: IPhase) => ReadonlyArray<string> =
getCustomParameterValuesByPhase();
Expand All @@ -58,7 +59,7 @@ function spliceShards(existingOperations: Set<Operation>, context: ICreateOperat
logFilenameIdentifier: baseLogFilenameIdentifier
} = operation;
if (phase && project && operationSettings?.sharding && !operation.runner) {
const { count: shards, shardOperationSettings } = operationSettings.sharding;
const { count: shards } = operationSettings.sharding;

/**
* A single operation to reduce the number of edges in the graph when creating shards.
Expand Down Expand Up @@ -165,6 +166,7 @@ function spliceShards(existingOperations: Set<Operation>, context: ICreateOperat
);
}

const projectConfiguration: RushProjectConfiguration | undefined = projectConfigurations.get(project);
for (let shard: number = 1; shard <= shards; shard++) {
const outputDirectory: string = outputFolderWithTemplate.replace(
TemplateStringRegexes.SHARD_INDEX,
Expand All @@ -175,7 +177,7 @@ function spliceShards(existingOperations: Set<Operation>, context: ICreateOperat
project,
phase,
settings: {
...shardOperationSettings,
...projectConfiguration?.operationSettingsByOperationName.get(shardOperationName),
operationName: shardOperationName,
outputFolderNames: [outputDirectory]
},
Expand Down
11 changes: 0 additions & 11 deletions libraries/rush-lib/src/schemas/rush-project.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -90,17 +90,6 @@
"outputFolderArgumentFormat": {
"type": "string",
"description": "The command-line argument to pass to the operation to specify the output folder. The string may contain the following placeholders: {phaseName} {shardIndex}. Must end with {shardIndex}. Defaults to --shard-output-folder=\".rush/operations/{phaseName}/shards/{shardIndex}\""
},
"shardOperationSettings": {
"type": "object",
"additionalProperties": false,
"properties": {
"weight": {
"description": "The number of concurrency units that this operation should take up. The maximum concurrency units is determined by the -p flag.",
"type": "integer",
"minimum": 0
}
}
}
}
},
Expand Down
Loading