[release/5.0] Add x64 RHEL7 and 8 builds. (#2175)

crummel · web-flow · commit 833dcf468265 · 2021-09-22T13:28:55.000-05:00
* Add x64 RHEL7 and 8 builds.

* Only clean apt-get when necessary.

* Address code review comment.

* Add tarball to local CI

* Bring forward release/3.1 fix for tarball path.

* Use profile and bashrc so we get the right clang and llvm paths.
diff --git a/.vsts.pipelines/builds/matrix.yml b/.vsts.pipelines/builds/matrix.yml
@@ -54,6 +54,7 @@ stages:
     name: debian9
     jobParameters:
       imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-d61254f-20190807161114
+      cleanAptGet: true
     matrix:
       Production: {}
       Online: { type: Online }
@@ -86,6 +87,7 @@ stages:
     name: ubuntu1804
     jobParameters:
       imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-source-build-20200401093015-ed32d26
+      cleanAptGet: true
 
 - template: ../stages/stage-jobs-matrix.yml
   parameters:
@@ -98,6 +100,38 @@ stages:
       scriptSuffix: .sh
       setupMac: true
 
+# RHEL builds only internal
+- ${{ if not(eq(variables['System.TeamProject'], 'public')) }}:
+  - template: ../stages/stage-jobs-matrix.yml
+    parameters:
+      jobTemplateName: ../jobs/ci-local.yml
+      name: rhel7_x64
+      jobParameters:
+        pool:
+          name: DotNet-SourceBuild-Internal
+          demands:
+          - Agent.OSArchitecture -equals X64
+          - rhel -equals 7
+        scriptPrefix: ./
+        scriptSuffix: .sh
+      matrix:
+        Offline: { type: Offline }
+  - template: ../stages/stage-jobs-matrix.yml
+    parameters:
+      jobTemplateName: ../jobs/ci-local.yml
+      name: rhel8_x64
+      jobParameters:
+        pool:
+          name: DotNet-SourceBuild-Internal
+          demands:
+          - Agent.OSArchitecture -equals X64
+          - rhel -equals 8
+        scriptPrefix: ./
+        scriptSuffix: .sh
+        useSystemLibunwind: true
+      matrix:
+        Offline: { type: Offline }
+
 # Prepare artifacts in all situations but public PR validation.
 - ${{ if not(and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'))) }}:
   - template: ../stages/prepare-artifacts.yml
@@ -108,4 +142,6 @@ stages:
       - job: centos8_Offline
       - job: debian9_Online
       - job: fedora31_Offline
+      - job: rhel7_x64_Offline
+      - job: rhel8_x64_Offline
       gatherPortableJob: centos71_Offline_Portable
diff --git a/.vsts.pipelines/jobs/ci-linux.yml b/.vsts.pipelines/jobs/ci-linux.yml
@@ -6,6 +6,7 @@ parameters:
   reportPrebuiltLeaks: false
   generatePrebuiltBurndown: false
   runUnitTests: false
+  cleanAptGet: false
 
 jobs:
 - job: ${{ parameters.job }}
@@ -52,6 +53,8 @@ jobs:
 
   steps:
   - template: ../steps/cleanup-unneeded-files.yml
+    parameters:
+      cleanAptGet: ${{ parameters.cleanAptGet }}
   - template: ../steps/docker-cleanup-linux.yml
   - template: ../steps/calculate-config-flags.yml
 
diff --git a/.vsts.pipelines/jobs/ci-linux_bootstrap.yml b/.vsts.pipelines/jobs/ci-linux_bootstrap.yml
@@ -3,6 +3,7 @@ parameters:
   pool:
     name: Hosted Ubuntu 1604
   imageName: null
+  cleanAptGet: false
 
 jobs:
 - job: ${{ parameters.job }}
@@ -30,6 +31,8 @@ jobs:
 
   steps:
   - template: ../steps/cleanup-unneeded-files.yml
+    parameters:
+      cleanAptGet: ${{ parameters.cleanAptGet }}
   - template: ../steps/docker-cleanup-linux.yml
 
   - task: DownloadBuildArtifacts@0
diff --git a/.vsts.pipelines/jobs/ci-local.yml b/.vsts.pipelines/jobs/ci-local.yml
@@ -1,46 +1,52 @@
 parameters:
   job: null
-  matrix:
-    Production: {}
   pool: null
   scriptPrefix: null
   scriptSuffix: null
   setupMac: false
   setupWindows: false
-  # Temporarily disable smoke-tests until entire product builds.
-  # See https://github.com/dotnet/source-build/issues/1552
-  skipSmokeTest: true
+  skipSmokeTest: false
   failOnPrebuiltBaselineError: true
+  reportPrebuiltLeaks: false
+  generatePrebuiltBurndown: false
+  runUnitTests: false
+  cleanAptGet: false
 
 jobs:
 - job: ${{ parameters.job }}
-  strategy:
-    matrix: ${{ parameters.matrix }}
   pool: ${{ parameters.pool }}
-  timeoutInMinutes: 240
+  timeoutInMinutes: 300
   variables:
+    rootDirectory: $(Build.SourcesDirectory)/..
+    tarballDirectory: $(rootDirectory)/tb
+    stagingDirectory: $(rootDirectory)/staging
+    dropDirectory: $(stagingDirectory)/drop
+    logsDirectory: $(Build.ArtifactStagingDirectory)/logs
+    tarballName: tarball_$(Build.BuildId)
     args.build: >-
       /p:Configuration=$(sb.configuration)
       /p:BuildPortableRuntime=$(sb.portable)
       /p:BuildPortableSdk=$(sb.portable)
       /p:FailOnPrebuiltBaselineError=$(failOnPrebuiltBaselineError)
       /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
       /p:AzDoPat=$(dn-bot-all-orgs-build-rw-code-rw)
+      /p:ArchiveDownloadedPackages=$(sb.tarball)
       $(tarballDownloadArgs)
     args.smokeTest: >
       --run-smoke-test
       /p:Configuration=$(sb.configuration)
       /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
     # Prefix to distinguish artifacts from different legs.
-    artifactName: ${{ format('{0} $(type)', parameters.job) }}
+    artifactName: ${{ parameters.job }}
     failOnPrebuiltBaselineError: ${{ parameters.failOnPrebuiltBaselineError }}
-    logsDirectory: $(Build.ArtifactStagingDirectory)/logs
+    reportPrebuiltLeaks: ${{ parameters.reportPrebuiltLeaks }}
+    systemLibunwind: ${{ parameters.systemLibunwind }}
     tarballDownloadArgs: >-
-      /p:SkipDownloadingReferencePackages=true
-      /p:SkipDownloadingPreviouslySourceBuiltPackages=true
+      /p:DownloadSourceBuildReferencePackagesTimeoutSeconds=600
+      /p:DownloadSourceBuiltArtifactsTimeoutSeconds=1500
     SOURCE_BUILD_SKIP_SUBMODULE_CHECK: true
     # Default type, can be overridden by matrix legs.
-    type: Production
+    type: ${{ coalesce(parameters.type, 'Production') }}
 
     ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}:
       SetInternalPackageFeedPatBashCommand: ''
@@ -56,6 +62,9 @@ jobs:
     clean: true
 
   - template: ../steps/cleanup-unneeded-files.yml
+    parameters:
+      cleanAptGet: ${{ parameters.cleanAptGet }}
+  - template: ../steps/calculate-config-flags.yml
 
   # Make sure submodules from other branches are removed: pass extra f.
   - script: git clean -xdff
@@ -79,11 +88,18 @@ jobs:
   - template: ../steps/check-space-powershell.yml
 
   # Build source-build.
-  - script: |
-      set -x
-      $(SetInternalPackageFeedPatBashCommand)
-      set -e
-      ${{ format('{0}build{1} $(args.build)', parameters.scriptPrefix, parameters.scriptSuffix) }}
+  # Use bash task here because it allows us to use the .bashrc, which sets up the correct
+  # paths for clang and llvm on RHEL machines.
+  - task: Bash@3
+    inputs:
+      targetType: inline
+      script: |
+        set -x
+        $(SetInternalPackageFeedPatBashCommand)
+        set -e
+        ${{ format('{0}build{1} $(args.build)', parameters.scriptPrefix, parameters.scriptSuffix) }}
+      noProfile: false
+      noRc: false
     displayName: Build source-build
     timeoutInMinutes: 150
 
@@ -98,6 +114,131 @@ jobs:
         ${{ format('{0}build{1} $(args.smokeTest)', parameters.scriptPrefix, parameters.scriptSuffix) }}
       displayName: Run smoke-test
 
+  # Create tarball.
+  - script: |
+      set -ex
+      df -h
+      args="--skip-prebuilt-check --skip-build --minimize-disk-usage"
+      if [ "$(reportPrebuiltLeaks)" = "true" ]; then
+        args="$args --enable-leak-detection"
+      fi
+      ./build-source-tarball.sh \
+        "$(tarballDirectory)/$(tarballName)" \
+        $args
+      du -h $(rootDirectory) | sort -h | tail -n 50
+    displayName: Create tarball
+    condition: and(succeeded(), eq(variables['sb.tarball'], true))
+
+  # Copy logs from the production build
+  - bash: |
+      set -ex
+      df -h
+      mkdir -p "$(logsDirectory)"
+      find . \( \
+        -path './artifacts/*-report/*' -o \
+        -path './artifacts/msbuild-debug/*' -o \
+        -path './artifacts/roslyn-debug/*' -o \
+        -path './artifacts/aspnet-debug/*' -o \
+        -iname '*.binlog' -o \
+        -iname '*.log' \) \
+        -exec cp {} --parents "$(logsDirectory)" \;
+      du -h $(rootDirectory) | sort -h | tail -n 50
+    displayName: Copy source-build production build logs
+    condition: always()
+    continueOnError: true
+
+  # Delete key directories from local copy of repo to save space
+  - bash: |
+      set -ex
+      df -h
+      sudo rm -rf $(rootDirectory)/source-build/artifacts/src
+      sudo rm -rf $(rootDirectory)/source-build/artifacts/obj
+      sudo rm -rf $(rootDirectory)/source-build/.git
+      sudo rm -rf $(rootDirectory)/source-build/packages
+      du -h $(rootDirectory) | sort -h | tail -n 50
+    displayName: Delete key directories from source copy of repo to save space
+
+  # tar the tarball directory into the drop directory.
+  - bash: |
+      set -ex
+      df -h
+      mkdir -p $(dropDirectory)/tarball/
+      smokeTestPackages="$(tarballDirectory)/$(tarballName)/packages/smoke-test-packages"
+      # smokeTestPackages is a package cache, with redundant data and unnecessary structure. E.g.
+      # $smokeTestPackages/name/version/name.version.nupkg <- We want this.
+      # $smokeTestPackages/name/version/lib/net46/name.dll <- This is already in the nupkg.
+      # This find moves the nupkg files into $smokeTestPackages:
+      find "$smokeTestPackages" -iname "*.nupkg" -exec mv {} "$smokeTestPackages" \;
+      # This find removes all non-nupkg files, which are not wanted:
+      find "$smokeTestPackages" -mindepth 1 -not -iname "*.nupkg" -delete
+      # Make one .tar.gz for build, another for extras necessary to smoke test:
+      tar --numeric-owner "--exclude=$smokeTestPackages" -zcf "$(dropDirectory)/tarball/$(tarballName).tar.gz" -C "$(tarballDirectory)" "$(tarballDirectory)/$(tarballName)"
+      tar --numeric-owner -zcf "$(dropDirectory)/tarball/$(tarballName)-smoke-test-prereqs.tar.gz" "$smokeTestPackages"
+      du -h $(rootDirectory) | sort -h | tail -n 50
+    displayName: Copy tarball to output
+    condition: and(succeeded(), eq(variables['sb.tarball'], true))
+
+  # Build tarball.
+  - task: Bash@3
+    inputs:
+      targetType: inline
+      script: |
+        set -ex
+        df -h
+        networkArg=
+        if [ "$(sb.tarballOffline)" = "true" ]; then
+          networkArg="sudo unshare -n"
+        fi
+        poisonArg=
+        if [ "$(reportPrebuiltLeaks)" = "true" ]; then
+          poisonArg="/p:EnablePoison=true"
+        fi
+        $networkArg "$(tarballDirectory)/$(tarballName)/build.sh" -- \
+          /p:Configuration=$(sb.configuration) \
+          /p:PortableBuild=$(sb.portable) \
+          /p:UseSystemLibunwind=$(systemLibunwind) \
+          /p:FailOnPrebuiltBaselineError=true \
+          $poisonArg
+        du -h $(rootDirectory) | sort -h | tail -n 50
+      noProfile: false
+      noRc: false
+    displayName: Build tarball
+    timeoutInMinutes: 150
+    condition: and(succeeded(), eq(variables['sb.tarball'], true))
+
+  # Run smoke tests.
+  - script: |
+      set -ex
+      df -h
+      sudo chown -R "$(id -u):$(id -g)" "$(tarballDirectory)/$(tarballName)"
+      "$(tarballDirectory)/$(tarballName)/build.sh" \
+        --run-smoke-test \
+        -- \
+        /p:Configuration=$(sb.configuration) \
+        /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
+      du -h $(rootDirectory) | sort -h | tail -n 50
+    displayName: Run smoke-test in tarball
+    condition: and(succeeded(), eq(variables['sb.tarball'], true))
+
+  - script: df -h
+    displayName: Check space (df -h)
+    condition: always()
+    continueOnError: true
+
+  # Copy source-built artifacts tarball to drop directory.
+  - bash: |
+      set -ex
+      df -h
+      mkdir -p "$(dropDirectory)/tarball/"
+      cd "$(tarballDirectory)/$(tarballName)"
+      find ./artifacts \( \
+        -iname 'Private.SourceBuilt.Artifacts*.tar.gz' \) \
+        -exec cp {} "$(dropDirectory)/tarball/" \;
+      du -h $(rootDirectory) | sort -h | tail -n 50
+    displayName: Copy source-built artifacts tarball
+    condition: eq(variables['sb.tarball'], true)
+    continueOnError: true
+
   # Gather artifacts. Uses git bash on Windows.
   - bash: |
       set -ex
@@ -123,11 +264,20 @@ jobs:
     continueOnError: true
 
   # Publish artifacts.
-  - task: PublishBuildArtifacts@1
+  - publish: $(logsDirectory)
+    artifact: Logs $(artifactName)
     displayName: Publish Logs artifact
     condition: always()
     continueOnError: true
-    inputs:
-      PathtoPublish: $(logsDirectory)
-      ArtifactName: Logs $(artifactName)
-      ArtifactType: Container
+  - publish: $(dropDirectory)/tarball
+    artifact: Tarball $(artifactName)
+    displayName: Publish Tarball artifact
+    condition: eq(variables['sb.tarball'], true)
+    continueOnError: true
+
+  - bash: |
+      set -ex
+      rm -rf "$(tarballDirectory)" "$(stagingDirectory)" "$(dropDirectory)" "$(logsDirectory)"
+    displayName: Cleanup extra staging directories
+    condition: always()
+    continueOnError: true
diff --git a/.vsts.pipelines/steps/cleanup-unneeded-files.yml b/.vsts.pipelines/steps/cleanup-unneeded-files.yml
@@ -1,10 +1,14 @@
 # Delete some files from the default image since we're tight on disk space.
 # List is from https://github.com/microsoft/azure-pipelines-image-generation/pull/1231#issuecomment-537174621
+parameters:
+- name: cleanAptGet
+  type: boolean
+  default: false
 
 steps:
 - script: sudo apt-get clean
   displayName: Clean apt-get cache
-  condition: ne(variables['Agent.OS'], 'Darwin')
+  condition: ${{ eq(parameters.cleanAptGet, true) }}
   continueOnError: true
 
 - script: sudo rm -rf /usr/local/lib/android
diff --git a/build-source-tarball.sh b/build-source-tarball.sh
