Rollup merge of rust-lang#65202 - pietroalbini:scriptify-ci-config, r=alexcrichton

ci: move most of the prepare config into scripts

This PR moves most of the configuration from the CI yamls into bash scripts, driven by a small Python script (which understands and emulates the two `##vso[` commands we use).

There are two reasons why we'd want to do this:

* Being able to prepare the build environment locally by just running `src/ci/prepare.py` simplifies a lot setting up a local VM similar to CI (software pre-installed in the CI images won't be prepared, but it's a start anyway).
* When we'll switch to GitHub Actions we'll need to either duplicate code in multiple workflows or write a preprocessor. Having all the prepare steps in a single one is going to simplify the implementation of both options.

Along with the move I did a few changes to the actual scripts:

* Mirrored all the remaining external URLs we download (except chocolatey) to the `rust-lang-ci-mirrors` bucket, to increase reliability and reduce the chance of supply chain attacks. I didn't audit and mirror the CI scripts outside this PR though.
* Extracted CI-specific behavior (like issuing `##vso[` commands and detecting the host platform) into `shared.sh` and included it in most of the scripts. This way a switch to another CI provider will be less painful.

It's possible (and easier) to review this commit-by-commit.
r? @alexcrichton
cc @rust-lang/infra

Author: Centril

src/ci/azure-pipelines/steps/install-clang.yml

@@ -48,86 +48,99 @@ steps:
 - bash: python src/ci/cpu-usage-over-time.py &> cpu-usage.csv &
   displayName: "Collect CPU-usage statistics in the background"
 
-- bash: printenv | sort
-  displayName: Show environment variables
+- bash: src/ci/scripts/dump-environment.sh
+  displayName: Show the current environment
 
-- bash: |
-    set -e
-    df -h
-    du . | sort -nr | head -n100
-  displayName: Show disk usage
-  # FIXME: this hasn't been tested, but maybe it works on Windows? Should test!
-  condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
-
-- template: install-sccache.yml
-- template: install-clang.yml
-
-# Switch to XCode 9.3 on OSX since it seems to be the last version that supports
-# i686-apple-darwin. We'll eventually want to upgrade this and it will probably
-# force us to drop i686-apple-darwin, but let's keep the wheels turning for now.
-- bash: |
-    set -e
-    sudo xcode-select --switch /Applications/Xcode_9.3.app
-  displayName: Switch to Xcode 9.3 (OSX)
-  condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
+- bash: src/ci/scripts/install-sccache.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  displayName: Install sccache
+  condition: and(succeeded(), not(variables.SKIP_JOB))
 
-- template: install-windows-build-deps.yml
+- bash: src/ci/scripts/install-clang.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  displayName: Install clang
+  condition: and(succeeded(), not(variables.SKIP_JOB))
 
-# Looks like docker containers have IPv6 disabled by default, so let's turn it
-# on since libstd tests require it
-- bash: |
-    set -e
-    sudo mkdir -p /etc/docker
-    echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json
-    sudo service docker restart
-  displayName: Enable IPv6
-  condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['Agent.OS'], 'Linux'))
+- bash: src/ci/scripts/switch-xcode.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  displayName: Switch to Xcode 9.3
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+
+- bash: src/ci/scripts/install-wix.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  displayName: Install wix
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+
+- bash: src/ci/scripts/install-innosetup.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  displayName: Install InnoSetup
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+
+- bash: src/ci/scripts/windows-symlink-build-dir.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  displayName: Ensure the build happens on C:\ instead of D:\
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+
+- bash: src/ci/scripts/disable-git-crlf-conversion.sh
+  displayName: "Disable git automatic line ending conversion (on C:/)"
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+
+- bash: src/ci/scripts/install-msys2.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+    SYSTEM_WORKFOLDER: $(System.Workfolder)
+  displayName: Install msys2
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+
+- bash: src/ci/scripts/install-mingw.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+    SYSTEM_WORKFOLDER: $(System.Workfolder)
+  displayName: Install MinGW
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+
+- bash: src/ci/scripts/install-ninja.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  displayName: Install ninja
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+
+- bash: src/ci/scripts/enable-docker-ipv6.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  displayName: Enable IPv6 on Docker
+  condition: and(succeeded(), not(variables.SKIP_JOB))
 
 # Disable automatic line ending conversion (again). On Windows, when we're
 # installing dependencies, something switches the git configuration directory or
 # re-enables autocrlf. We've not tracked down the exact cause -- and there may
 # be multiple -- but this should ensure submodules are checked out with the
 # appropriate line endings.
-- bash: git config --replace-all --global core.autocrlf false
-  displayName: "Disable git automatic line ending conversion"
+- bash: src/ci/scripts/disable-git-crlf-conversion.sh
+  displayName: Disable git automatic line ending conversion
+  condition: and(succeeded(), not(variables.SKIP_JOB))
 
-# Check out all our submodules, but more quickly than using git by using one of
-# our custom scripts
-- bash: |
-    set -e
-    mkdir -p $HOME/rustsrc
-    $BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc
-  condition: and(succeeded(), not(variables.SKIP_JOB), ne(variables['Agent.OS'], 'Windows_NT'))
-  displayName: Check out submodules (Unix)
-- script: |
-    if not exist C:\cache\rustsrc\NUL mkdir C:\cache\rustsrc
-    sh src/ci/init_repo.sh . /c/cache/rustsrc
-  condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['Agent.OS'], 'Windows_NT'))
-  displayName: Check out submodules (Windows)
-
-# See also the disable for autocrlf above, this just checks that it worked
-#
-# We check both in rust-lang/rust and in a submodule to make sure both are
-# accurate. Submodules are checked out significantly later than the main
-# repository in this script, so settings can (and do!) change between then.
-#
-# Linux (and maybe macOS) builders don't currently have dos2unix so just only
-# run this step on Windows.
-- bash: |
-    set -x
-    # print out the git configuration so we can better investigate failures in
-    # the following
-    git config --list --show-origin
-    dos2unix -ih Cargo.lock src/tools/rust-installer/install-template.sh
-    endings=$(dos2unix -ic Cargo.lock src/tools/rust-installer/install-template.sh)
-    # if endings has non-zero length, error out
-    if [ -n "$endings" ]; then exit 1 ; fi
-  condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
-  displayName: Verify line endings are LF
+- bash: src/ci/scripts/checkout-submodules.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  displayName: Checkout submodules
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+
+- bash: src/ci/scripts/verify-line-endings.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  displayName: Verify line endings
+  condition: and(succeeded(), not(variables.SKIP_JOB))
 
 # Ensure the `aws` CLI is installed so we can deploy later on, cache docker
 # images, etc.
-- bash: src/ci/install-awscli.sh
+- bash: src/ci/scripts/install-awscli.sh
   env:
     AGENT_OS: $(Agent.OS)
   condition: and(succeeded(), not(variables.SKIP_JOB))

src/ci/azure-pipelines/steps/install-sccache.yml

@@ -117,7 +117,7 @@ make check-bootstrap
 
 # Display the CPU and memory information. This helps us know why the CI timing
 # is fluctuating.
-if isOSX; then
+if isMacOS; then
     system_profiler SPHardwareDataType || true
     sysctl hw || true
     ncpus=$(sysctl -n hw.ncpu)

src/ci/azure-pipelines/steps/install-windows-build-deps.yml

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Check out all our submodules, but more quickly than using git by using one of
+# our custom scripts
+
+set -euo pipefail
+IFS=$'\n\t'
+
+source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
+
+if isWindows; then
+    path="/c/cache/rustsrc"
+else
+    path="${HOME}/rustsrc"
+fi
+
+mkdir -p "${path}"
+"$(cd "$(dirname "$0")" && pwd)/../init_repo.sh" . "${path}"

src/ci/azure-pipelines/steps/run.yml

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Disable automatic line ending conversion, which is enabled by default on
+# Azure's Windows image. Having the conversion enabled caused regressions both
+# in our test suite (it broke miri tests) and in the ecosystem, since we
+# started shipping install scripts with CRLF endings instead of the old LF.
+#
+# Note that we do this a couple times during the build as the PATH and current
+# user/directory change, e.g. when mingw is enabled.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+git config --replace-all --global core.autocrlf false