Skip to content

Commit 9f85d05

Browse files
wtgodbewtgodbe
authored
Split helix into subsets (#62644)
* Try splitting helix into subsets * Fix spacing * Update a doc * Fix comment
1 parent 31e0db6 commit 9f85d05

File tree

4 files changed

+145
-55
lines changed

4 files changed

+145
-55
lines changed

‎.azure/pipelines/ci-public.yml

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -570,11 +570,11 @@ stages:
570570
publishOnError: true
571571
includeForks: true
572572

573-
# Helix x64
573+
# Helix x64 subset 1
574574
- template: jobs/default-build.yml
575575
parameters:
576-
jobName: Helix_x64
577-
jobDisplayName: 'Tests: Helix x64'
576+
jobName: Helix_x64_Subset_1
577+
jobDisplayName: 'Tests: Helix x64 Subset 1'
578578
agentOs: Windows
579579
timeoutInMinutes: 240
580580
steps:
@@ -587,14 +587,43 @@ stages:
587587
- script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -noBuildRepoTasks -noBuildNative -noBuild -test
588588
-projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true
589589
/p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
590-
/p:VsTestUseMSBuildOutput=false /p:RunTemplateTests=false
590+
/p:VsTestUseMSBuildOutput=false /p:RunTemplateTests=false /p:HelixSubset=1
591591
displayName: Run build.cmd helix target
592592
env:
593593
HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
594594
SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
595595

596596
artifacts:
597-
- name: Helix_Logs_Attempt_$(System.JobAttempt)
597+
- name: Helix_Subset_1_Logs_Attempt_$(System.JobAttempt)
598+
path: artifacts/log/
599+
publishOnError: true
600+
includeForks: true
601+
602+
# Helix x64 subset 2
603+
- template: jobs/default-build.yml
604+
parameters:
605+
jobName: Helix_x64_Subset_2
606+
jobDisplayName: 'Tests: Helix x64 Subset 2'
607+
agentOs: Windows
608+
timeoutInMinutes: 240
609+
steps:
610+
# Build the shared framework
611+
- script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64
612+
/p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
613+
/p:VsTestUseMSBuildOutput=false
614+
displayName: Build shared fx
615+
# -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step.
616+
- script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -noBuildRepoTasks -noBuildNative -noBuild -test
617+
-projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true
618+
/p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
619+
/p:VsTestUseMSBuildOutput=false /p:RunTemplateTests=false /p:HelixSubset=2
620+
displayName: Run build.cmd helix target
621+
env:
622+
HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
623+
SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
624+
625+
artifacts:
626+
- name: Helix_Subset_2_Logs_Attempt_$(System.JobAttempt)
598627
path: artifacts/log/
599628
publishOnError: true
600629
includeForks: true

‎.azure/pipelines/ci.yml

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -624,8 +624,8 @@ extends:
624624
# Helix x64
625625
- template: .azure/pipelines/jobs/default-build.yml@self
626626
parameters:
627-
jobName: Helix_x64
628-
jobDisplayName: 'Tests: Helix x64'
627+
jobName: Helix_x64_Subset_1
628+
jobDisplayName: 'Tests: Helix x64 Subset 1'
629629
agentOs: Windows
630630
timeoutInMinutes: 240
631631
steps:
@@ -637,15 +637,44 @@ extends:
637637
displayName: Build shared fx
638638
# -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step.
639639
- script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -noBuildRepoTasks -noBuildNative -noBuild -test
640-
-projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true
640+
-projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true /p:HelixSubset=1
641641
/p:CrossgenOutput=false /p:RunTemplateTests=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
642642
displayName: Run build.cmd helix target
643643
env:
644644
HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
645645
SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
646646

647647
artifacts:
648-
- name: Helix_Logs_Attempt_$(System.JobAttempt)
648+
- name: Helix_Logs_Subset_1_Attempt_$(System.JobAttempt)
649+
path: artifacts/log/
650+
publishOnError: true
651+
includeForks: true
652+
653+
# Helix x64
654+
- template: .azure/pipelines/jobs/default-build.yml@self
655+
parameters:
656+
jobName: Helix_x64_Subset_2
657+
jobDisplayName: 'Tests: Helix x64 Subset 2'
658+
agentOs: Windows
659+
timeoutInMinutes: 240
660+
steps:
661+
# Build the shared framework
662+
- script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64
663+
/p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
664+
env:
665+
MSBUILDUSESERVER: "1"
666+
displayName: Build shared fx
667+
# -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step.
668+
- script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -noBuildRepoTasks -noBuildNative -noBuild -test
669+
-projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true /p:HelixSubset=2
670+
/p:CrossgenOutput=false /p:RunTemplateTests=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
671+
displayName: Run build.cmd helix target
672+
env:
673+
HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
674+
SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
675+
676+
artifacts:
677+
- name: Helix_Logs_Subset_2_Attempt_$(System.JobAttempt)
649678
path: artifacts/log/
650679
publishOnError: true
651680
includeForks: true
@@ -747,7 +776,8 @@ extends:
747776
- Windows_Test
748777
- MacOS_Test
749778
- Linux_Test
750-
- Helix_x64
779+
- Helix_x64_Subset_1
780+
- Helix_x64_Subset_2
751781
- ${{ if eq(variables.enableSourceIndex, 'true') }}:
752782
- SourceIndexStage1
753783
- Source_Build_Managed

‎docs/AddingNewProjects.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,13 @@ Sample PR of final result: https://github.com/dotnet/aspnetcore/pull/41945
2828
- If VS has not already modified these files, open the `.slnf` you want to add the project to. Create a solution folder for your project if doesn't exist already. Then right click solution folder -> Add -> Existing Project... -> follow the wizard.
2929
1. Run the `eng/scripts/GenerateProjectList.ps1` file to regenerate a number of `eng/*.props` files e.g. ProjectReferences.props.
3030

31-
**Note:** If you are adding a new project to the root `src` directory, you will also need to add a reference in both of the `DotNetProjects` lists of the `eng/Build.props` file. The first list (the one with condition `'$(BuildMainlyReferenceProviders)' != 'true'"`) has items in the format of:
31+
**Note:** If you are adding a new project to the root `src` directory, you will also need to add a reference in both the `ProjectsWithTestsSubsetN` and `DotNetProjects` lists of the `eng/Build.props` file. The `ProjectsWithTestsSubsetN` lists (the one with condition `'$(BuildMainlyReferenceProviders)' != 'true'"`) has items in the format of:
3232
```XML
33-
<DotNetProjects Include="
33+
<ProjectsWithTestsSubsetN Include="
3434
$(RepoRoot)src\[YOUR FOLDER]\**\*.csproj;
3535
...
3636
```
37-
while the second (the one with condition `'$(BuildMainlyReferenceProviders)' == 'true'"`) has them in the format of (note the second `src`):
37+
. You should add your project to whichever `ProjectsWithTestsSubsetN` list is shorter (`ProjectsWithTestsSubset1` or `ProjectsWithTestsSubset2`). The `DotNetProjects` list (the one with condition `'$(BuildMainlyReferenceProviders)' == 'true'"`) has them in the format of (note the second `src`):
3838
```XML
3939
<DotNetProjects Include="
4040
$(RepoRoot)src\[YOUR FOLDER]\**\src\*.csproj;

‎eng/Build.props

Lines changed: 73 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -150,56 +150,23 @@
150150
Use caution to avoid deep recursion. If the globbing pattern picks up something which exceeds MAX_PATH,
151151
the entire pattern will silently fail to evaluate correctly.
152152
-->
153+
154+
<!--
155+
When adding new projects to this file, add them to either ProjectsWithTestsSubset1 or
156+
ProjectsWithTestsSubset2 - whichever has fewer entries.
157+
-->
158+
159+
<!-- Strictly projects with no associated tests - projects with tests are added in subsets below. -->
153160
<DotNetProjects Include="
161+
$(RepoRoot)src\Assets\**\*.*proj;
154162
$(RepoRoot)src\Framework\App.Ref\src\Microsoft.AspNetCore.App.Ref.sfxproj;
155163
$(RepoRoot)src\Framework\App.Ref.Internal\src\Microsoft.AspNetCore.App.Ref.Internal.csproj;
156164
$(RepoRoot)src\Framework\App.Runtime\src\aspnetcore-runtime.proj;
157165
$(RepoRoot)src\Framework\App.Runtime\src\aspnetcore-runtime-composite.proj;
158166
$(RepoRoot)src\Framework\App.Runtime\src\aspnetcore-base-runtime.proj;
159167
$(RepoRoot)src\Framework\App.Runtime\bundle\aspnetcore-runtime-bundle.bundleproj;
160-
$(RepoRoot)src\Framework\AspNetCoreAnalyzers\test\Microsoft.AspNetCore.App.Analyzers.Test.csproj;
161-
$(RepoRoot)src\Framework\test\Microsoft.AspNetCore.App.UnitTests.csproj;
162-
$(RepoRoot)src\Assets\**\*.*proj;
163-
$(RepoRoot)src\Caching\**\*.*proj;
164-
$(RepoRoot)src\DefaultBuilder\**\*.*proj;
165-
$(RepoRoot)src\Features\**\*.*proj;
166-
$(RepoRoot)src\DataProtection\**\*.*proj;
167-
$(RepoRoot)src\Antiforgery\**\*.*proj;
168-
$(RepoRoot)src\Hosting\**\*.*proj;
169-
$(RepoRoot)src\Http\**\*.*proj;
170-
$(RepoRoot)src\HttpClientFactory\**\*.*proj;
171-
$(RepoRoot)src\Html.Abstractions\**\*.*proj;
172-
$(RepoRoot)src\Identity\**\*.*proj;
173-
$(RepoRoot)src\Servers\**\*.csproj;
174-
$(RepoRoot)src\Security\**\*.*proj;
175-
$(RepoRoot)src\SiteExtensions\Microsoft.Web.Xdt.Extensions\**\*.csproj;
176-
$(RepoRoot)src\SiteExtensions\LoggingAggregate\test\**\*.csproj;
177-
$(RepoRoot)src\Shared\**\*.*proj;
178-
$(RepoRoot)src\Tools\**\*.*proj;
179-
$(RepoRoot)src\Logging.AzureAppServices\**\src\*.csproj;
180-
$(RepoRoot)src\Middleware\**\*.csproj;
181-
$(RepoRoot)src\Razor\**\*.*proj;
182-
$(RepoRoot)src\Mvc\**\*.*proj;
183-
$(RepoRoot)src\Azure\**\*.*proj;
184-
$(RepoRoot)src\SignalR\**\*.csproj;
185-
$(RepoRoot)src\StaticAssets\**\*.csproj;
186-
$(RepoRoot)src\Components\**\*.csproj;
187-
$(RepoRoot)src\Analyzers\**\*.csproj;
188-
$(RepoRoot)src\FileProviders\**\*.csproj;
189-
$(RepoRoot)src\Configuration.KeyPerFile\**\*.csproj;
190-
$(RepoRoot)src\Localization\**\*.csproj;
191-
$(RepoRoot)src\ObjectPool\**\*.csproj;
192-
$(RepoRoot)src\JSInterop\**\*.csproj;
193-
$(RepoRoot)src\WebEncoders\**\*.csproj;
194-
$(RepoRoot)src\HealthChecks\**\*.csproj;
195-
$(RepoRoot)src\Testing\**\*.csproj;
196-
$(RepoRoot)src\Grpc\**\*.csproj;
197-
$(RepoRoot)src\ProjectTemplates\**\*.csproj;
198-
$(RepoRoot)src\Extensions\**\*.csproj;
199-
$(RepoRoot)src\BuildAfterTargetingPack\*.csproj;
200-
$(RepoRoot)src\OpenApi\**\*.csproj;
201-
$(RepoRoot)src\Validation\**\*.csproj;
202168
$(RepoRoot)eng\tools\HelixTestRunner\HelixTestRunner.csproj;
169+
$(RepoRoot)src\BuildAfterTargetingPack\*.csproj;
203170
"
204171
Exclude="
205172
@(ProjectToBuild);
@@ -253,6 +220,70 @@
253220
$(RepoRoot)**\obj\**\*;"
254221
Condition=" '$(BuildMainlyReferenceProviders)' == 'true' " />
255222

223+
224+
<ProjectsWithTestsSubset1 Include="
225+
$(RepoRoot)src\Framework\AspNetCoreAnalyzers\test\Microsoft.AspNetCore.App.Analyzers.Test.csproj;
226+
$(RepoRoot)src\Framework\test\Microsoft.AspNetCore.App.UnitTests.csproj;
227+
$(RepoRoot)src\Caching\**\*.*proj;
228+
$(RepoRoot)src\DefaultBuilder\**\*.*proj;
229+
$(RepoRoot)src\Features\**\*.*proj;
230+
$(RepoRoot)src\DataProtection\**\*.*proj;
231+
$(RepoRoot)src\Antiforgery\**\*.*proj;
232+
$(RepoRoot)src\Hosting\**\*.*proj;
233+
$(RepoRoot)src\Http\**\*.*proj;
234+
$(RepoRoot)src\HttpClientFactory\**\*.*proj;
235+
$(RepoRoot)src\Html.Abstractions\**\*.*proj;
236+
$(RepoRoot)src\Identity\**\*.*proj;
237+
$(RepoRoot)src\Servers\**\*.csproj;
238+
$(RepoRoot)src\Security\**\*.*proj;
239+
$(RepoRoot)src\SiteExtensions\Microsoft.Web.Xdt.Extensions\**\*.csproj;
240+
$(RepoRoot)src\SiteExtensions\LoggingAggregate\test\**\*.csproj;
241+
$(RepoRoot)src\Shared\**\*.*proj;
242+
$(RepoRoot)src\Tools\**\*.*proj;
243+
$(RepoRoot)src\Logging.AzureAppServices\**\src\*.csproj;
244+
$(RepoRoot)src\Middleware\**\*.csproj;
245+
"
246+
Exclude="
247+
@(ProjectToBuild);
248+
@(ProjectToExclude);
249+
$(RepoRoot)**\node_modules\**\*;
250+
$(RepoRoot)**\bin\**\*;
251+
$(RepoRoot)**\obj\**\*;"
252+
Condition=" '$(BuildMainlyReferenceProviders)' != 'true' " />
253+
254+
<ProjectsWithTestsSubset2 Include="
255+
$(RepoRoot)src\Razor\**\*.*proj;
256+
$(RepoRoot)src\Mvc\**\*.*proj;
257+
$(RepoRoot)src\Azure\**\*.*proj;
258+
$(RepoRoot)src\SignalR\**\*.csproj;
259+
$(RepoRoot)src\StaticAssets\**\*.csproj;
260+
$(RepoRoot)src\Components\**\*.csproj;
261+
$(RepoRoot)src\Analyzers\**\*.csproj;
262+
$(RepoRoot)src\FileProviders\**\*.csproj;
263+
$(RepoRoot)src\Configuration.KeyPerFile\**\*.csproj;
264+
$(RepoRoot)src\Localization\**\*.csproj;
265+
$(RepoRoot)src\ObjectPool\**\*.csproj;
266+
$(RepoRoot)src\JSInterop\**\*.csproj;
267+
$(RepoRoot)src\WebEncoders\**\*.csproj;
268+
$(RepoRoot)src\HealthChecks\**\*.csproj;
269+
$(RepoRoot)src\Testing\**\*.csproj;
270+
$(RepoRoot)src\Grpc\**\*.csproj;
271+
$(RepoRoot)src\ProjectTemplates\**\*.csproj;
272+
$(RepoRoot)src\Extensions\**\*.csproj;
273+
$(RepoRoot)src\OpenApi\**\*.csproj;
274+
$(RepoRoot)src\Validation\**\*.csproj;
275+
"
276+
Exclude="
277+
@(ProjectToBuild);
278+
@(ProjectToExclude);
279+
$(RepoRoot)**\node_modules\**\*;
280+
$(RepoRoot)**\bin\**\*;
281+
$(RepoRoot)**\obj\**\*;"
282+
Condition=" '$(BuildMainlyReferenceProviders)' != 'true' " />
283+
284+
<DotNetProjects Condition=" '$(HelixSubset)' == '' OR '$(HelixSubset)' == '1'" Include="@(ProjectsWithTestsSubset1)" />
285+
<DotNetProjects Condition=" '$(HelixSubset)' == '' OR '$(HelixSubset)' == '2'" Include="@(ProjectsWithTestsSubset2)" />
286+
256287
<ProjectToBuild Condition=" '$(BuildManaged)' == 'true'" Include="@(DotNetProjects)" Exclude="@(ProjectToExclude)" BuildStep="managed" />
257288
<ProjectToExclude Condition=" '$(BuildManaged)' != 'true'" Include="@(DotNetProjects)" />
258289
</ItemGroup>

0 commit comments

Comments
 (0)