From d4d18797788b3650876034d9bf14addcd0307721 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Wed, 9 Jul 2025 12:38:36 -0700 Subject: [PATCH 1/4] Try splitting helix into subsets --- .azure/pipelines/ci-public.yml | 39 +++++++++-- .azure/pipelines/ci.yml | 40 ++++++++++-- eng/Build.props | 114 +++++++++++++++++++++------------ 3 files changed, 141 insertions(+), 52 deletions(-) diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index f075662ae69d..7cf5ea808e5a 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -570,11 +570,11 @@ stages: publishOnError: true includeForks: true - # Helix x64 + # Helix x64 subset 1 - template: jobs/default-build.yml parameters: - jobName: Helix_x64 - jobDisplayName: 'Tests: Helix x64' + jobName: Helix_x64_Subset_1 + jobDisplayName: 'Tests: Helix x64 Subset 1' agentOs: Windows timeoutInMinutes: 240 steps: @@ -587,14 +587,43 @@ stages: - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -noBuildRepoTasks -noBuildNative -noBuild -test -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) - /p:VsTestUseMSBuildOutput=false /p:RunTemplateTests=false + /p:VsTestUseMSBuildOutput=false /p:RunTemplateTests=false /p:HelixSubset=1 displayName: Run build.cmd helix target env: HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops artifacts: - - name: Helix_Logs_Attempt_$(System.JobAttempt) + - name: Helix_Subset_1_Logs_Attempt_$(System.JobAttempt) + path: artifacts/log/ + publishOnError: true + includeForks: true + + # Helix x64 subset 2 + - template: jobs/default-build.yml + parameters: + jobName: Helix_x64_Subset_2 + jobDisplayName: 'Tests: Helix x64 Subset 2' + agentOs: Windows + timeoutInMinutes: 240 + steps: + # Build the shared framework + - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64 + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + /p:VsTestUseMSBuildOutput=false + displayName: Build shared fx + # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. + - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -noBuildRepoTasks -noBuildNative -noBuild -test + -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + /p:VsTestUseMSBuildOutput=false /p:RunTemplateTests=false /p:HelixSubset=2 + displayName: Run build.cmd helix target + env: + HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues + SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + + artifacts: + - name: Helix_Subset_2_Logs_Attempt_$(System.JobAttempt) path: artifacts/log/ publishOnError: true includeForks: true diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index b1dfbc20d544..ab0d36e1bedf 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -624,8 +624,8 @@ extends: # Helix x64 - template: .azure/pipelines/jobs/default-build.yml@self parameters: - jobName: Helix_x64 - jobDisplayName: 'Tests: Helix x64' + jobName: Helix_x64_Subset_1 + jobDisplayName: 'Tests: Helix x64 Subset 1' agentOs: Windows timeoutInMinutes: 240 steps: @@ -637,7 +637,7 @@ extends: displayName: Build shared fx # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -noBuildRepoTasks -noBuildNative -noBuild -test - -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true + -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true /p:HelixSubset=1 /p:CrossgenOutput=false /p:RunTemplateTests=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) displayName: Run build.cmd helix target env: @@ -645,7 +645,36 @@ extends: SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops artifacts: - - name: Helix_Logs_Attempt_$(System.JobAttempt) + - name: Helix_Logs_Subset_1_Attempt_$(System.JobAttempt) + path: artifacts/log/ + publishOnError: true + includeForks: true + + # Helix x64 + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Helix_x64_Subset_2 + jobDisplayName: 'Tests: Helix x64 Subset 2' + agentOs: Windows + timeoutInMinutes: 240 + steps: + # Build the shared framework + - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64 + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + env: + MSBUILDUSESERVER: "1" + displayName: Build shared fx + # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. + - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -noBuildRepoTasks -noBuildNative -noBuild -test + -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true /p:HelixSubset=2 + /p:CrossgenOutput=false /p:RunTemplateTests=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + displayName: Run build.cmd helix target + env: + HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues + SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + + artifacts: + - name: Helix_Logs_Subset_2_Attempt_$(System.JobAttempt) path: artifacts/log/ publishOnError: true includeForks: true @@ -747,7 +776,8 @@ extends: - Windows_Test - MacOS_Test - Linux_Test - - Helix_x64 + - Helix_x64_Subset_1 + - Helix_x64_Subset_2 - ${{ if eq(variables.enableSourceIndex, 'true') }}: - SourceIndexStage1 - Source_Build_Managed diff --git a/eng/Build.props b/eng/Build.props index b447810b064b..a371c5216800 100644 --- a/eng/Build.props +++ b/eng/Build.props @@ -150,56 +150,22 @@ Use caution to avoid deep recursion. If the globbing pattern picks up something which exceeds MAX_PATH, the entire pattern will silently fail to evaluate correctly. --> + + + + + + + + + + + From 675d6b3d0a86c02e0e4ee287434c16c4c1b00d92 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Wed, 9 Jul 2025 12:44:28 -0700 Subject: [PATCH 2/4] Fix spacing --- eng/Build.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Build.props b/eng/Build.props index a371c5216800..4e75f2d875a7 100644 --- a/eng/Build.props +++ b/eng/Build.props @@ -280,8 +280,8 @@ $(RepoRoot)**\obj\**\*;" Condition=" '$(BuildMainlyReferenceProviders)' != 'true' " /> - - + + From 04ca9e3c2e9e5ecdf632a3131e4d741847ced7ed Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Wed, 9 Jul 2025 15:19:26 -0700 Subject: [PATCH 3/4] Update a doc --- docs/AddingNewProjects.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/AddingNewProjects.md b/docs/AddingNewProjects.md index cfb9a5e5a3ab..ce1a25cc1ede 100644 --- a/docs/AddingNewProjects.md +++ b/docs/AddingNewProjects.md @@ -28,13 +28,13 @@ Sample PR of final result: https://github.com/dotnet/aspnetcore/pull/41945 - If VS has not already modified these files, open the `.slnf` you want to add the project to. Create a solution folder for your project if doesn't exist already. Then right click solution folder -> Add -> Existing Project... -> follow the wizard. 1. Run the `eng/scripts/GenerateProjectList.ps1` file to regenerate a number of `eng/*.props` files e.g. ProjectReferences.props. -**Note:** If you are adding a new project to the root `src` directory, you will also need to add a reference in both of the `DotNetProjects` lists of the `eng/Build.props` file. The first list (the one with condition `'$(BuildMainlyReferenceProviders)' != 'true'"`) has items in the format of: +**Note:** If you are adding a new project to the root `src` directory, you will also need to add a reference in both the `ProjectsWithTestsSubsetN` and `DotNetProjects` lists of the `eng/Build.props` file. The `ProjectsWithTestsSubsetN` lists (the one with condition `'$(BuildMainlyReferenceProviders)' != 'true'"`) has items in the format of: ```XML -