[release/8.0] Switch to dSAS for internal runtimes (dotnet#56488)

* Update arcade

* Switch to dSAS for internal runtimes

* Enable internal sources for source build

* Fixup for publishing in arcade 8.0

* Fix spaciong

Author: mmitche

.azure/pipelines/ci.yml

@@ -101,7 +101,6 @@ variables:
     value: /bl:artifacts/log/Release/Build.Installers.binlog
   - name: WindowsArm64InstallersLogArgs
     value: /bl:artifacts/log/Release/Build.Installers.Arm64.binlog
-- group: DotNetBuilds storage account read tokens
 - name: _InternalRuntimeDownloadArgs
   value: -RuntimeSourceFeed https://dotnetbuilds.blob.core.windows.net/internal
          -RuntimeSourceFeedKey $(dotnetbuilds-internal-container-read-token-base64)
@@ -675,6 +674,7 @@ extends:
       # Source build
       - template: /eng/common/templates-official/job/source-build.yml@self
         parameters:
+          enableInternalSources: true
           platform:
             name: 'Managed'
             container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9'

.azure/pipelines/jobs/default-build.yml

@@ -210,7 +210,6 @@ jobs:
                 # Include the variables we always want.
                 COMPlus_DbgEnableMiniDump: 1
                 COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
-                DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token)
                 # Expand provided `env:` properties, if any.
                 ${{ if step.env }}:
                   ${{ step.env }}
@@ -222,14 +221,12 @@ jobs:
             env:
               COMPlus_DbgEnableMiniDump: 1
               COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
-              DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token)
         - ${{ if ne(parameters.agentOs, 'Windows') }}:
           - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs)
             displayName: Run build.sh
             env:
               COMPlus_DbgEnableMiniDump: 1
               COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
-              DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token)
 
       - ${{ parameters.afterBuild }}
 
@@ -441,6 +438,20 @@ jobs:
           env:
             Token: $(dn-bot-dnceng-artifact-feeds-rw)
 
+      # Populates internal runtime SAS tokens.
+      - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml
+
+      # Populate dotnetbuilds-internal base64 sas tokens.
+      - template: /eng/common/templates-official/steps/get-delegation-sas.yml
+        parameters:
+          federatedServiceConnection: 'dotnetbuilds-internal-read'
+          outputVariableName: 'dotnetbuilds-internal-container-read-token'
+          expiryInHours: 1
+          base64Encode: false
+          storageAccount: dotnetbuilds
+          container: internal
+          permissions: rl
+
       # Add COMPlus_* environment variables to build steps.
       - ${{ if ne(parameters.steps, '')}}:
         - ${{ each step in parameters.steps }}:

eng/Version.Details.xml

@@ -376,26 +376,26 @@
       <Uri>https://github.com/dotnet/winforms</Uri>
       <Sha>abda8e3bfa78319363526b5a5f86863ec979940e</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24266.3">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24352.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e6f70c7dd528f05cd28cec2a179d58c22e91d9ac</Sha>
+      <Sha>8b879da4e449c48d99f3f642fc429379a64e8fe8</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="8.0.0-beta.24266.3">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="8.0.0-beta.24352.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e6f70c7dd528f05cd28cec2a179d58c22e91d9ac</Sha>
+      <Sha>8b879da4e449c48d99f3f642fc429379a64e8fe8</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Templating" Version="8.0.0-beta.24266.3">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Templating" Version="8.0.0-beta.24352.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e6f70c7dd528f05cd28cec2a179d58c22e91d9ac</Sha>
+      <Sha>8b879da4e449c48d99f3f642fc429379a64e8fe8</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.24266.3">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.24352.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e6f70c7dd528f05cd28cec2a179d58c22e91d9ac</Sha>
+      <Sha>8b879da4e449c48d99f3f642fc429379a64e8fe8</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version="8.0.0-beta.24266.3">
+    <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version="8.0.0-beta.24352.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e6f70c7dd528f05cd28cec2a179d58c22e91d9ac</Sha>
+      <Sha>8b879da4e449c48d99f3f642fc429379a64e8fe8</Sha>
     </Dependency>
     <Dependency Name="Microsoft.Extensions.Diagnostics.Testing" Version="8.1.0-preview.23604.1">
       <Uri>https://github.com/dotnet/extensions</Uri>

eng/Versions.props

@@ -162,9 +162,9 @@
     <NuGetVersioningVersion>6.2.4</NuGetVersioningVersion>
     <NuGetFrameworksVersion>6.2.4</NuGetFrameworksVersion>
     <!-- Packages from dotnet/arcade -->
-    <MicrosoftDotNetBuildTasksInstallersVersion>8.0.0-beta.24266.3</MicrosoftDotNetBuildTasksInstallersVersion>
-    <MicrosoftDotNetBuildTasksTemplatingVersion>8.0.0-beta.24266.3</MicrosoftDotNetBuildTasksTemplatingVersion>
-    <MicrosoftDotNetRemoteExecutorVersion>8.0.0-beta.24266.3</MicrosoftDotNetRemoteExecutorVersion>
+    <MicrosoftDotNetBuildTasksInstallersVersion>8.0.0-beta.24352.1</MicrosoftDotNetBuildTasksInstallersVersion>
+    <MicrosoftDotNetBuildTasksTemplatingVersion>8.0.0-beta.24352.1</MicrosoftDotNetBuildTasksTemplatingVersion>
+    <MicrosoftDotNetRemoteExecutorVersion>8.0.0-beta.24352.1</MicrosoftDotNetRemoteExecutorVersion>
     <!-- Packages from dotnet/source-build-externals -->
     <MicrosoftSourceBuildIntermediatesourcebuildexternalsVersion>8.0.0-alpha.1.24269.1</MicrosoftSourceBuildIntermediatesourcebuildexternalsVersion>
     <!-- Packages from dotnet/source-build-reference-packages -->

eng/common/post-build/publish-using-darc.ps1

@@ -2,7 +2,6 @@ param(
   [Parameter(Mandatory=$true)][int] $BuildId,
   [Parameter(Mandatory=$true)][int] $PublishingInfraVersion,
   [Parameter(Mandatory=$true)][string] $AzdoToken,
-  [Parameter(Mandatory=$true)][string] $MaestroToken,
   [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
   [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
@@ -31,13 +30,13 @@ try {
   }
 
   & $darc add-build-to-channel `
-  --id $buildId `
-  --publishing-infra-version $PublishingInfraVersion `
-  --default-channels `
-  --source-branch main `
-  --azdev-pat $AzdoToken `
-  --bar-uri $MaestroApiEndPoint `
-  --password $MaestroToken `
+    --id $buildId `
+    --publishing-infra-version $PublishingInfraVersion `
+    --default-channels `
+    --source-branch main `
+    --azdev-pat "$AzdoToken" `
+    --bar-uri "$MaestroApiEndPoint" `
+    --ci `
 	@optionalParams
 
   if ($LastExitCode -ne 0) {

eng/common/templates-official/job/publish-build-assets.yml

@@ -76,13 +76,16 @@ jobs:
 
     - task: NuGetAuthenticate@1
 
-    - task: PowerShell@2
+    - task: AzureCLI@2
       displayName: Publish Build Assets
       inputs:
-        filePath: eng\common\sdk-task.ps1
-        arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
+        azureSubscription: "Darc: Maestro Production"
+        scriptType: ps
+        scriptLocation: scriptPath
+        scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+        arguments: >
+          -task PublishBuildAssets -restore -msbuildEngine dotnet
           /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
-          /p:BuildAssetRegistryToken=$(MaestroAccessToken)
           /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
           /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
           /p:OfficialBuildId=$(Build.BuildNumber)
@@ -144,7 +147,6 @@ jobs:
           arguments: -BuildId $(BARBuildId) 
             -PublishingInfraVersion 3
             -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
-            -MaestroToken '$(MaestroApiAccessToken)'
             -WaitPublishingFinish true
             -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
             -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/templates-official/job/source-build.yml

@@ -31,6 +31,12 @@ parameters:
   #   container and pool.
   platform: {}
 
+  # If set to true and running on a non-public project,
+  # Internal blob storage locations will be enabled.
+  # This is not enabled by default because many repositories do not need internal sources
+  # and do not need to have the required service connections approved in the pipeline.
+  enableInternalSources: false
+
 jobs:
 - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
   displayName: Source-Build (${{ parameters.platform.name }})
@@ -62,6 +68,8 @@ jobs:
     clean: all
 
   steps:
+  - ${{ if eq(parameters.enableInternalSources, true) }}:
+    - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml
   - template: /eng/common/templates-official/steps/source-build.yml
     parameters:
       platform: ${{ parameters.platform }}

eng/common/templates-official/job/source-index-stage1.yml

@@ -23,7 +23,7 @@ jobs:
     value: ${{ parameters.sourceIndexPackageSource }}
   - name: BinlogPath
     value: ${{ parameters.binlogPath }}
-  - template: /eng/common/templates/variables/pool-providers.yml
+  - template: /eng/common/templates-official/variables/pool-providers.yml
 
   ${{ if ne(parameters.pool, '') }}:
     pool: ${{ parameters.pool }}
@@ -34,7 +34,8 @@ jobs:
         demands: ImageOverride -equals windows.vs2019.amd64.open
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         name: $(DncEngInternalBuildPool)
-        demands: ImageOverride -equals windows.vs2019.amd64
+        image: windows.vs2022.amd64
+        os: windows
 
   steps:
   - ${{ each preStep in parameters.preSteps }}:
@@ -70,16 +71,13 @@ jobs:
         scriptType: 'ps'
         scriptLocation: 'inlineScript'
         inlineScript: |
-          echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId"
-          echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken"
-          echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId"
+          echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId"
+          echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken"
+          echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId"
 
     - script: |
-        echo "Client ID: $(ARM_CLIENT_ID)"
-        echo "ID Token: $(ARM_ID_TOKEN)"
-        echo "Tenant ID: $(ARM_TENANT_ID)"
         az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
       displayName: "Login to Azure"
 
     - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
-      displayName: Upload stage1 artifacts to source index
+      displayName: Upload stage1 artifacts to source index

eng/common/templates-official/jobs/source-build.yml

@@ -21,6 +21,12 @@ parameters:
   # one job runs on 'defaultManagedPlatform'.
   platforms: []
 
+  # If set to true and running on a non-public project,
+  # Internal nuget and blob storage locations will be enabled.
+  # This is not enabled by default because many repositories do not need internal sources
+  # and do not need to have the required service connections approved in the pipeline.
+  enableInternalSources: false
+
 jobs:
 
 - ${{ if ne(parameters.allCompletedJobId, '') }}:
@@ -38,9 +44,11 @@ jobs:
     parameters:
       jobNamePrefix: ${{ parameters.jobNamePrefix }}
       platform: ${{ platform }}
+      enableInternalSources: ${{ parameters.enableInternalSources }}
 
 - ${{ if eq(length(parameters.platforms), 0) }}:
   - template: /eng/common/templates-official/job/source-build.yml
     parameters:
       jobNamePrefix: ${{ parameters.jobNamePrefix }}
       platform: ${{ parameters.defaultManagedPlatform }}
+      enableInternalSources: ${{ parameters.enableInternalSources }}

eng/common/templates-official/post-build/post-build.yml

@@ -272,14 +272,16 @@ stages:
 
         - task: NuGetAuthenticate@1
 
-        - task: PowerShell@2
+        - task: AzureCLI@2
           displayName: Publish Using Darc
           inputs:
-            filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+            azureSubscription: "Darc: Maestro Production"
+            scriptType: ps
+            scriptLocation: scriptPath
+            scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
             arguments: -BuildId $(BARBuildId) 
               -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
               -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
-              -MaestroToken '$(MaestroApiAccessToken)'
               -WaitPublishingFinish true
               -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
               -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'