Merge pull request #903 from dagood/windows/2.1

Fix build on Windows up to Core-Setup, add CI leg

Author: dagood

.vsts.pipelines/builds/ci-internal.yml

@@ -0,0 +1,4 @@
+jobs:
+- template: matrix.yml
+  parameters:
+    windowsPoolName: dotnet-internal-temp

.vsts.pipelines/builds/ci.yml

@@ -1,43 +1,2 @@
-# Matrix build types:
-#   Production: Only runs the build, not a tarball build. Called "production" because it's the part
-#     used to produce tarballs. Some platforms might not be capable of producing a tarball yet: we
-#     still call this type of build "production" for consistency.
-#   Online: The leg produces a tarball then builds it while connected to the internet.
-#   Offline: The leg produces a tarball then builds it offline. Network disconnected using Docker.
-
 jobs:
-- template: ../jobs/ci-linux.yml
-  parameters:
-    job: centos71
-    imageName: microsoft/dotnet-buildtools-prereqs:centos-7-b46d863-20180719033416
-    reportPrebuiltLeaks: true
-    matrix:
-      Production: {}
-      Online: { type: Online }
-      Offline: { type: Offline }
-      Offline Portable: { type: Offline Portable }
-
-- template: ../jobs/ci-linux.yml
-  parameters:
-    job: debian82
-    imageName: microsoft/dotnet-buildtools-prereqs:debian-8.2-4e6e894-20181023212416
-    matrix:
-      Production: {}
-      Online: { type: Online }
-
-- template: ../jobs/ci-linux.yml
-  parameters:
-    job: fedora29
-    imageName: microsoft/dotnet-buildtools-prereqs:fedora-29-2f0798a-20181105183801
-    matrix:
-      Production: {}
-      Online: { type: Online }
-      Offline: { type: Offline }
-      Offline Portable: { type: Offline Portable }
-
-- template: ../jobs/ci-linux.yml
-  parameters:
-    job: ubuntu1604
-    imageName: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-c103199-20180628134544
-
-- template: ../jobs/ci-osx.yml
+- template: matrix.yml

.vsts.pipelines/builds/matrix.yml

@@ -0,0 +1,62 @@
+# Matrix build types:
+#   Production: Only runs the build, not a tarball build. Called "production" because it's the part
+#     used to produce tarballs. Some platforms might not be capable of producing a tarball yet: we
+#     still call this type of build "production" for consistency.
+#   Online: The leg produces a tarball then builds it while connected to the internet.
+#   Offline: The leg produces a tarball then builds it offline. Network disconnected using Docker.
+
+parameters:
+  windowsPoolName: dotnet-external-temp
+
+jobs:
+- template: ../jobs/ci-linux.yml
+  parameters:
+    job: centos71
+    imageName: microsoft/dotnet-buildtools-prereqs:centos-7-b46d863-20180719033416
+    reportPrebuiltLeaks: true
+    matrix:
+      Production: {}
+      Online: { type: Online }
+      Offline: { type: Offline }
+      Offline Portable: { type: Offline Portable }
+
+- template: ../jobs/ci-linux.yml
+  parameters:
+    job: debian82
+    imageName: microsoft/dotnet-buildtools-prereqs:debian-8.2-4e6e894-20181023212416
+    matrix:
+      Production: {}
+      Online: { type: Online }
+
+- template: ../jobs/ci-linux.yml
+  parameters:
+    job: fedora29
+    imageName: microsoft/dotnet-buildtools-prereqs:fedora-29-2f0798a-20181105183801
+    matrix:
+      Production: {}
+      Online: { type: Online }
+      Offline: { type: Offline }
+      Offline Portable: { type: Offline Portable }
+
+- template: ../jobs/ci-linux.yml
+  parameters:
+    job: ubuntu1604
+    imageName: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-c103199-20180628134544
+
+- template: ../jobs/ci-local.yml
+  parameters:
+    job: osx
+    pool:
+      name: Hosted macOS
+    buildScript: ./build.sh
+    setupMac: true
+
+- template: ../jobs/ci-local.yml
+  parameters:
+    job: windows
+    pool:
+      name: ${{ parameters.windowsPoolName }}
+    buildScript: build.cmd
+    setupWindows: true
+    skipSmokeTest: true
+    failOnPrebuiltBaselineError: false

.vsts.pipelines/jobs/ci-linux.yml

@@ -38,7 +38,7 @@ jobs:
 
   steps:
   - template: ../steps/docker-cleanup-linux.yml
-  - template: ../steps/calculate-config-flags-linux.yml
+  - template: ../steps/calculate-config-flags.yml
 
   # Create working directory and copy source into it.
   - script: |
@@ -50,7 +50,7 @@ jobs:
         cp -r . /root/sb/source-build'
     displayName: Clean sb directory and copy source from cloned directory
 
-  - template: ../steps/init-submodules-sh.yml
+  - template: ../steps/init-submodules.yml
     parameters:
       commandPrefix: $(docker.run) $(docker.src.map) $(docker.src.work) $(imageName)
 

.vsts.pipelines/jobs/ci-local.yml

@@ -0,0 +1,99 @@
+parameters:
+  job: null
+  matrix:
+    Production: {}
+  pool: null
+  buildScript: null
+  setupMac: false
+  setupWindows: false
+  skipSmokeTest: false
+  failOnPrebuiltBaselineError: true
+
+jobs:
+- job: ${{ parameters.job }}
+  strategy:
+    matrix: ${{ parameters.matrix }}
+  pool: ${{ parameters.pool }}
+  timeoutInMinutes: 240
+  variables:
+    args.build: >
+      /p:Configuration=$(sb.configuration)
+      /p:PortableBuild=$(sb.portable)
+      /p:FailOnPrebuiltBaselineError=$(failOnPrebuiltBaselineError)
+      /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
+    args.smokeTest: >
+      /t:RunSmokeTest
+      /p:Configuration=$(sb.configuration)
+      /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
+    # Prefix to distinguish artifacts from different legs.
+    artifactName: ${{ format('{0} $(type)', parameters.job) }}
+    failOnPrebuiltBaselineError: ${{ parameters.failOnPrebuiltBaselineError }}
+    logsDirectory: $(Build.ArtifactStagingDirectory)/logs
+    SOURCE_BUILD_SKIP_SUBMODULE_CHECK: true
+    # Default type, can be overridden by matrix legs.
+    type: Production
+
+  steps:
+  - checkout: self
+    clean: true
+
+  # Make sure submodules from other branches are removed: pass extra f.
+  - script: git clean -xdff
+    displayName: Clean leftover submodules
+
+  - template: ../steps/check-space-powershell.yml
+
+  - ${{ if eq(parameters.setupMac, true) }}:
+    - template: ../steps/setup-macos-native.yml
+  - ${{ if eq(parameters.setupWindows, true) }}:
+    - template: ../steps/setup-windows-bash.yml
+
+  - template: ../steps/calculate-config-flags.yml
+
+  - template: ../steps/init-submodules.yml
+
+  - template: ../steps/check-space-powershell.yml
+
+  # Build source-build.
+  - script: ${{ format('{0} $(args.build)', parameters.buildScript) }}
+    displayName: Build source-build
+    timeoutInMinutes: 150
+
+  - template: ../steps/check-space-powershell.yml
+
+  # Run smoke tests.
+  - ${{ if ne(parameters.skipSmokeTest, true) }}:
+    - bash: ${{ format('{0} $(args.smokeTest)', parameters.buildScript) }}
+      displayName: Run smoke-test
+
+  # Gather artifacts. Uses git bash on Windows.
+  - bash: |
+      set -x
+      copyWithParents="cp {} --parents"
+      if command -v rsync; then
+        # On Mac, "--parents" isn't supported, but we can use rsync.
+        copyWithParents="rsync -R {}"
+      fi
+      # Clean up previous build's staging dir, in case VSTS didn't do it.
+      rm -rf "$(logsDirectory)"
+      # Copy all these files to the logs dir, preserving relative path.
+      mkdir -p "$(logsDirectory)"
+      find . \( \
+        -path './bin/*-report/*' -o \
+        -path './bin/msbuild-debug/*' -o \
+        -iname '*.binlog' -o \
+        -iname '*.log' \) \
+        -exec $copyWithParents "$(logsDirectory)" \;
+    displayName: Copy logs to staging directory
+    condition: always()
+    continueOnError: true
+
+  # Publish artifacts.
+  - task: PublishBuildArtifacts@1
+    displayName: Publish Logs artifact
+    condition: always()
+    continueOnError: true
+    inputs:
+      PathtoPublish: $(logsDirectory)
+      ArtifactName: Logs $(artifactName)
+      ArtifactType: Container

.vsts.pipelines/jobs/ci-osx.yml

@@ -1,5 +1,5 @@
 steps:
-- script: |
+- bash: |
     setvariable() {
       echo "Setting '$1' to '$2'"
       echo "##vso[task.setvariable variable=$1;]$2"

.vsts.pipelines/steps/calculate-config-flags-linux.yml renamed to .vsts.pipelines/steps/calculate-config-flags.yml

@@ -0,0 +1,9 @@
+steps:
+- powershell: |
+    $g = 1024 * 1024 * 1024;
+    echo "Name`tAvailable (GiB)`tTotal (GiB)"
+    [System.IO.DriveInfo]::GetDrives() |
+      %{ echo "$($_.Name)`t$($_.AvailableFreeSpace / $g)`t$($_.TotalSize / $g)" }
+  displayName: (Diag) Check drive fullness
+  condition: always()
+  continueOnError: true

.vsts.pipelines/steps/check-space-powershell.yml

@@ -3,7 +3,7 @@ parameters:
 
 steps:
   # Fetch vsts commits if building internally.
-  - script: |
+  - bash: |
       set -x
       # Ignore failure for the first command. It will intentionally fail if the commit is only
       # available in VSTS. "submodule update --init" is the simplest way to set up the submodule
@@ -17,7 +17,7 @@ steps:
     condition: and(succeeded(), ne(variables['user.PAT'], ''))
 
   # Initialize submodules.
-  - script: |
+  - bash: |
       set -x
       $commandPrefix git submodule update --init --recursive
     displayName: Initialize submodules

.vsts.pipelines/steps/init-submodules-sh.yml renamed to .vsts.pipelines/steps/init-submodules.yml

@@ -0,0 +1,21 @@
+steps:
+- script: |
+    setvariable() {
+      (
+        set +x
+        echo "Setting '$1' to '$2'"
+        echo "##vso[task.setvariable variable=$1;]$2"
+      )
+    }
+    set -x
+    # Install macOS native dependencies not present in hosted pool.
+    # Based on https://github.com/sbomer/coreclr/blob/dc2fa1b96415f87d694e5b6e751e38ca6fbd09a1/unix-pipeline.yml#L36-L39
+    brew install icu4c openssl
+    brew link --force icu4c
+    # Follow https://github.com/dotnet/corefx/blob/master/Documentation/building/unix-instructions.md#user-content-macos
+    brew install cmake pkgconfig openssl
+    brew_openssl_paths=$(brew info openssl | cut -d ' ' -f 1 | grep '^/.*/openssl/')
+    # Use 'for' to get last/latest openssl install dir.
+    for openssl_path in $brew_openssl_paths; do :; done
+    setvariable OPENSSL_ROOT_DIR "$openssl_path"
+  displayName: Install native dependencies