[main] Update common Docker engineering infrastructure with latest

eng/common/templates/1es-official.yml

@@ -2,7 +2,7 @@
 # do the following:
 #
 # - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines
-# - The versions repo resource must be named `InternalVersionsRepo` or `PublicVersionsRepo` to avoid SDL scans
+# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans
 # - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans
 #
 # If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in
@@ -57,8 +57,7 @@ extends:
           enabled: true
         sourceRepositoriesToScan:
           exclude:
-          - repository: InternalVersionsRepo
-          - repository: PublicVersionsRepo
+          - repository: VersionsRepo
         sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
         tsa:
           enabled: true

eng/common/templates/jobs/publish.yml

@@ -5,6 +5,11 @@ parameters:
   customPublishVariables: []
   sourceBuildPipelineDefinitionId: ""
   sourceBuildPipelineRunId: ""
+  versionsRepoRef: null
+  versionsRepoPath: ""
+  # When true, overrides the commit SHA in merged image info files to use the current repository commit.
+  # This ensures that updated images reference the correct commit in their commitUrl properties.
+  overrideImageInfoCommit: false
 
 jobs:
 - job: Publish
@@ -31,16 +36,28 @@ jobs:
     value: $(artifactsPath)/imageInfo
   - name: sourceBuildIdOutputDir
     value: $(Build.ArtifactStagingDirectory)/sourceBuildId
+  - name: commitOverrideArg
+    ${{ if eq(parameters.overrideImageInfoCommit, true) }}:
+      value: --commit-override $(Build.SourceVersion)
+    ${{ else }}:
+      value: ''
   - ${{ parameters.customPublishVariables }}
 
   steps:
+  - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
+    parameters:
+      cloneVersionsRepo: ${{ variables.publishImageInfo }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+
   - template: /eng/common/templates/steps/retain-build.yml@self
 
   - template: /eng/common/templates/steps/init-docker-linux.yml@self
 
   - pwsh: |
       $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI
       echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName"
+      $versionsRepoRoot = "$(Pipeline.Workspace)/s/${{ parameters.versionsRepoPath }}"
+      echo "##vso[task.setvariable variable=versionsRepoRoot]$versionsRepoRoot"
     displayName: Set Publish Variables
 
   - ${{ parameters.customInitSteps }}
@@ -138,13 +155,16 @@ jobs:
   - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data
     displayName: Create EOL Annotation Data Directory
 
-  - powershell: >-
-      $(engCommonPath)/Invoke-WithRetry.ps1
-      "curl -fSL
-      --output $(imageInfoHostDir)/full-image-info-orig.json
-      https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/refs/heads/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath)"
+  - script: |-
+      cd $(versionsRepoRoot)
+      git pull origin $(gitHubVersionsRepoInfo.branch)
+    condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
+    displayName: Pull Latest Changes from Versions Repo
+
+  - script: >-
+      cp $(versionsRepoRoot)/$(gitHubImageInfoVersionsPath) $(imageInfoHostDir)/full-image-info-orig.json
     condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
-    displayName: Download Latest Image Info
+    displayName: Copy Latest Image Info from Versions Repo
 
   - script: >
       $(runImageBuilderCmd) mergeImageInfo
@@ -155,6 +175,7 @@ jobs:
       --manifest $(manifest)
       --publish
       --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json
+      $(commitOverrideArg)
     condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
     displayName: Merge Image Info
 

eng/common/templates/stages/build-and-test.yml

@@ -22,8 +22,7 @@ parameters:
   internalProjectName: null
   publicProjectName: null
 
-  internalVersionsRepoRef: null
-  publicVersionsRepoRef: null
+  versionsRepoRef: ""
 
   isInternalServicingValidation: false
 
@@ -88,11 +87,9 @@ stages:
       noCache: ${{ parameters.noCache }}
       customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
@@ -102,11 +99,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -120,11 +115,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -138,11 +131,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -156,11 +147,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -174,11 +163,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -192,17 +179,14 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
       isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
@@ -212,11 +196,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -266,11 +248,9 @@ stages:
         customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
         sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
         commonInitStepsForMatrixAndBuild:
-        - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
           parameters:
-            noCache: ${{ parameters.noCache }}
-            internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-            publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+            versionsRepoRef: ${{ parameters.versionsRepoRef }}
     - template: /eng/common/templates/jobs/test-images-linux-client.yml@self
       parameters:
         name: Linux_amd64

eng/common/templates/stages/dotnet/build-and-test.yml

@@ -30,6 +30,8 @@ parameters:
   internalProjectName: null
   publicProjectName: null
 
+  versionsRepoRef: null
+
 stages:
 - template: /eng/common/templates/stages/build-and-test.yml@self
   parameters:
@@ -51,8 +53,9 @@ stages:
     testMatrixType: ${{ parameters.testMatrixType }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
 
-    internalVersionsRepoRef: InternalVersionsRepo
-    publicVersionsRepoRef: PublicVersionsRepo
+    # Only clone versions repo if we need to reference it during the build in order to cache images.
+    ${{ if eq(parameters.noCache, false) }}:
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
     # Linux AMD64
     linuxAmd64Pool:

eng/common/templates/stages/dotnet/build-test-publish-repo.yml

@@ -32,6 +32,7 @@ parameters:
   # Other common parameters
   internalProjectName: null
   publicProjectName: null
+  versionsRepoRef: null
 
 
 stages:
@@ -61,6 +62,7 @@ stages:
     # Other
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
 - template: /eng/common/templates/stages/dotnet/publish.yml@self
   parameters:
@@ -70,3 +72,4 @@ stages:
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}

eng/common/templates/stages/dotnet/publish.yml

@@ -10,6 +10,7 @@ parameters:
   customPublishInitSteps: []
   sourceBuildPipelineDefinitionId: ''
   sourceBuildPipelineRunId: ''
+  versionsRepoRef: null
 
 stages:
 - template: /eng/common/templates/stages/publish.yml@self
@@ -20,6 +21,7 @@ stages:
     isStandalonePublish: ${{ parameters.isStandalonePublish }}
     sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
     customPublishInitSteps:
     - pwsh: |

eng/common/templates/stages/publish.yml

@@ -14,6 +14,15 @@ parameters:
   sourceBuildPipelineDefinitionId: ''
   sourceBuildPipelineRunId: ''
 
+  versionsRepoRef: null
+  versionsRepoPath: "versions"
+
+  # When true, any updated images will have the SHA in their commit URL updated
+  # to the commit that this pipeline is running on, instead of the commit they
+  # were built from. Use in combination with isStandalonePublish to ensure that
+  # internally built images still reference public Dockerfiles.
+  overrideImageInfoCommit: false
+
 ################################################################################
 # Publish Images
 ################################################################################
@@ -65,3 +74,6 @@ stages:
         customInitSteps: ${{ parameters.customPublishInitSteps }}
         sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }}
         sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+        versionsRepoRef: ${{ parameters.versionsRepoRef }}
+        versionsRepoPath: ${{ parameters.versionsRepoPath }}
+        overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }}

eng/common/templates/steps/init-matrix-build-publish.yml

@@ -0,0 +1,77 @@
+# Initialize common variables used in
+# - Generating build matrix
+# - Building images
+# - Running tests
+# - Publishing images
+
+parameters:
+  versionsRepoRef: ""
+  versionsRepoPath: "versions"
+
+steps:
+- checkout: self
+- ${{ if ne(parameters.versionsRepoRef, '') }}:
+  - checkout: ${{ parameters.versionsRepoRef }}
+    path: s/${{ parameters.versionsRepoPath }}
+    persistCredentials: true
+    fetchDepth: 1
+    condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
+- powershell: |
+    $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)"
+    if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest") {
+      $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix $(mirrorRepoPrefix) --registry-override $(acr-staging.server)"
+    }
+
+    if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") {
+      $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'"
+    }
+
+    if ("${{ parameters.versionsRepoRef }}" -ne "") {
+      $versionsBasePath = "${{ parameters.versionsRepoPath }}/"
+      $pipelineDisabledCache = "false"
+
+      $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/")
+      if ($pathSeparatorIndex -ge 0) {
+        $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1)
+      }
+      else {
+        $buildRepoName = "$(Build.Repository.Name)"
+      }
+
+      $engCommonPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engCommonRelativePath)"
+
+      $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng"
+      $manifest = "$buildRepoName/$(manifest)"
+      $testResultsDirectory = "$buildRepoName/$testResultsDirectory"
+
+      if ("$(testScriptPath)") {
+        $testScriptPath = "$buildRepoName/$(testScriptPath)"
+      }
+
+      echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName"
+      echo "##vso[task.setvariable variable=engCommonPath]$engCommonPath"
+      echo "##vso[task.setvariable variable=manifest]$manifest"
+      echo "##vso[task.setvariable variable=engPath]$engPath"
+      echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath"
+      echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory"
+    }
+    else {
+      $versionsBasePath = ""
+      $pipelineDisabledCache = "true"
+    }
+
+    echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions"
+    echo "##vso[task.setvariable variable=versionsBasePath]$versionsBasePath"
+    echo "##vso[task.setvariable variable=pipelineDisabledCache]$pipelineDisabledCache"
+  displayName: Set Common Variables for Matrix, Build, and Publish
+
+- ${{ if ne(parameters.versionsRepoRef, '') }}:
+  # Special logic is needed to copy the tsaoptions.json file to a well known location for the 1ES PT.
+  # This template has multiple checkouts and AzDO doesn't have support for dynamically determining the
+  # default repo path therefore the 1es-official logic can't calculate the repo's tsa config file path.
+  - task: CopyFiles@2
+    displayName: Copy TSA Config
+    inputs:
+      SourceFolder: '$(Build.Repository.LocalPath)/$(buildRepoName)'
+      Contents: '.config/tsaoptions.json'
+      TargetFolder: '$(Build.SourcesDirectory)'

eng/common/templates/variables/docker-images.yml

@@ -1,5 +1,5 @@
 variables:
-  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2749906
+  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2757304
   imageNames.imageBuilder: $(imageNames.imageBuilderName)
   imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId)
   imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner