Auto merge of servo#29954 - mrobinson:windows-bootstrap, r=mukilan

Windows bootstrap support

<!-- Please describe your changes on the following line: -->

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] These changes fix servo#25224
- [x] These changes do not require tests because they are just support script changes.

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

Author: bors-servo

README.md

@@ -43,9 +43,12 @@ manually, try the [manual build setup][manual-build].
 
  - Download and run [`rustup-init.exe`](https://win.rustup.rs/) then follow the onscreen instructions.
  - Install [chocolatey](https://chocolatey.org/)
- - Run `choco install support\windows\chocolatey.config`
-   *This will install CMake, Git, LLVM, Ninja, NuGet, Python and the Visual Studio 2019 Build Tools.*
- - Run `mach bootstrap-gstreamer`
+ - Run `mach bootstrap`
+  - *This will install CMake, Git, Ninja, NuGet, Python and the Visual Studio 2019 Build Tools
+     via choco in an Administrator console. It can take quite a while.*
+  - *If you already have Visual Studio 2019 installed, this may not install all necessary components.
+     Please follow the Visual Studio 2019 installation instructions in the [manual setup][manual-build].*
+- Run `refreshenv`
 
 See also [Windows Troubleshooting Tips][windows-tips].
 
@@ -174,4 +177,4 @@ There are lots of mach commands you can use. You can list them with `./mach
 The generated documentation can be found on https://doc.servo.org/servo/index.html
 
 [manual-build]: https://github.com/servo/servo/wiki/Building#manual-build-setup
-[windows-tips]: https://github.com/servo/servo/wiki/Building#troubleshooting-the-windows-build
+[windows-tips]: https://github.com/servo/servo/wiki/Building#troubleshooting-the-windows-build

python/mach_bootstrap.py

@@ -241,13 +241,8 @@ def bootstrap_command_only(topdir):
     import servo.platform
     import servo.util
 
-    # We are not set up yet, so we always use the default cache directory
-    # for the initial bootstrap.
-    # TODO(mrobinson): Why not just run the bootstrap command in this case?
-
     try:
-        servo.platform.get().bootstrap(
-            servo.util.get_default_cache_dir(topdir), '-f' in sys.argv)
+        servo.platform.get().bootstrap('-f' in sys.argv or '--force' in sys.argv)
     except NotImplementedError as exception:
         print(exception)
         return 1

python/servo/bootstrap_commands.py

@@ -46,7 +46,7 @@ def bootstrap(self, force=False):
         # ./mach bootstrap calls mach_bootstrap.bootstrap_command_only so that
         # it can install dependencies without needing mach's dependencies
         try:
-            servo.platform.get().bootstrap(self.context.sharedir, force)
+            servo.platform.get().bootstrap(force)
         except NotImplementedError as exception:
             print(exception)
             return 1
@@ -60,7 +60,7 @@ def bootstrap(self, force=False):
                      help='Boostrap without confirmation')
     def bootstrap_gstreamer(self, force=False):
         try:
-            servo.platform.get().bootstrap_gstreamer(self.context.sharedir, force)
+            servo.platform.get().bootstrap_gstreamer(force)
         except NotImplementedError as exception:
             print(exception)
             return 1

python/servo/build_commands.py

@@ -178,11 +178,11 @@ def build(self, release=False, dev=False, jobs=None, params=None, no_package=Fal
 
             # Override any existing GStreamer installation with the vendored libraries.
             env["GSTREAMER_1_0_ROOT_" + arch['gst']] = path.join(
-                self.msvc_package_dir("gstreamer-uwp"), arch['gst_root']
+                servo.platform.windows.get_dependency_dir("gstreamer-uwp"), arch['gst_root']
             )
             env["PKG_CONFIG_PATH"] = path.join(
-                self.msvc_package_dir("gstreamer-uwp"), arch['gst_root'],
-                "lib", "pkgconfig"
+                servo.platform.windows.get_dependency_dir("gstreamer-uwp"),
+                arch['gst_root'], "lib", "pkgconfig"
             )
 
         if 'windows' in host:

python/servo/command_base.py

@@ -476,8 +476,7 @@ def get_nightly_binary_path(self, nightly_date):
         return self.get_executable(destination_folder)
 
     def msvc_package_dir(self, package):
-        return path.join(self.context.sharedir, "msvc-dependencies", package,
-                         servo.platform.windows.DEPENDENCIES[package])
+        return servo.platform.windows.get_dependency_dir(package)
 
     def vs_dirs(self):
         assert 'windows' in servo.platform.host_triple()
@@ -508,11 +507,7 @@ def build_env(self, is_build=False):
         extra_path = []
         effective_target = self.cross_compile_target or servo.platform.host_triple()
         if "msvc" in effective_target:
-            extra_path += [path.join(self.msvc_package_dir("cmake"), "bin")]
             extra_path += [path.join(self.msvc_package_dir("llvm"), "bin")]
-            extra_path += [path.join(self.msvc_package_dir("ninja"), "bin")]
-            extra_path += [self.msvc_package_dir("nuget")]
-
             env.setdefault("CC", "clang-cl.exe")
             env.setdefault("CXX", "clang-cl.exe")
             if self.is_uwp_build:
@@ -911,10 +906,7 @@ def ensure_bootstrapped(self, rustup_components=None):
         if self.context.bootstrapped:
             return
 
-        # Always check if all needed MSVC dependencies are installed
-        target_platform = self.cross_compile_target or servo.platform.host_triple()
-        if "msvc" in target_platform:
-            Registrar.dispatch("bootstrap", context=self.context)
+        servo.platform.get().passive_bootstrap()
 
         if self.config["tools"]["use-rustup"]:
             self.ensure_rustup_version()

python/servo/platform/base.py

@@ -75,7 +75,7 @@ def library_path_variable_name(self):
     def executable_suffix(self):
         return ""
 
-    def _platform_bootstrap(self, _cache_dir: str, _force: bool) -> bool:
+    def _platform_bootstrap(self, _force: bool) -> bool:
         raise NotImplementedError("Bootstrap installation detection not yet available.")
 
     def _platform_bootstrap_gstreamer(self, _force: bool) -> bool:
@@ -97,11 +97,17 @@ def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> boo
             == 0
         )
 
-    def bootstrap(self, cache_dir: str, force: bool):
-        if not self._platform_bootstrap(cache_dir, force):
+    def bootstrap(self, force: bool):
+        if not self._platform_bootstrap(force):
             print("Dependencies were already installed!")
 
-    def bootstrap_gstreamer(self, _cache_dir: str, force: bool):
+    def passive_bootstrap(self) -> bool:
+        """A bootstrap method that is called without explicitly invoking `./mach bootstrap`
+           but that is executed in the process of other `./mach` commands. This should be
+           as fast as possible."""
+        return False
+
+    def bootstrap_gstreamer(self, force: bool):
         if not self._platform_bootstrap_gstreamer(force):
             root = self.gstreamer_root(None)
             if root:

python/servo/platform/linux.py

@@ -94,7 +94,7 @@ def get_distro_and_version() -> Tuple[str, str]:
 
         return (distrib, version)
 
-    def _platform_bootstrap(self, _cache_dir: str, force: bool) -> bool:
+    def _platform_bootstrap(self, force: bool) -> bool:
         if self.distro.lower() == 'nixos':
             print('NixOS does not need bootstrap, it will automatically enter a nix-shell')
             print('Just run ./mach build')

python/servo/platform/macos.py

@@ -54,7 +54,7 @@ def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> boo
             return False
         return True
 
-    def _platform_bootstrap(self, _cache_dir: str, force: bool) -> bool:
+    def _platform_bootstrap(self, _force: bool) -> bool:
         installed_something = False
         try:
             brewfile = os.path.join(util.SERVO_ROOT, "etc", "homebrew", "Brewfile")

python/servo/platform/windows.py

@@ -8,25 +8,19 @@
 # except according to those terms.
 
 import os
-import shutil
 import subprocess
 import tempfile
 from typing import Optional
 import urllib
 import zipfile
-from distutils.version import LooseVersion
 
-import six
 from .. import util
 from .base import Base
 
 DEPS_URL = "https://github.com/servo/servo-build-deps/releases/download/msvc-deps/"
 DEPENDENCIES = {
-    "cmake": "3.14.3",
     "llvm": "15.0.5",
     "moztools": "3.2",
-    "ninja": "1.7.1",
-    "nuget": "08-08-2019",
     "openssl": "111.3.0+1.1.1c-vs2017-2019-09-18",
     "gstreamer-uwp": "1.16.0.5",
     "openxr-loader-uwp": "1.0",
@@ -38,6 +32,11 @@
 DEPENDENCIES_DIR = os.path.join(util.get_target_dir(), "dependencies")
 
 
+def get_dependency_dir(package):
+    """Get the directory that a given Windows dependency should extract to."""
+    return os.path.join(DEPENDENCIES_DIR, package, DEPENDENCIES[package])
+
+
 class Windows(Base):
     def __init__(self, triple: str):
         super().__init__(triple)
@@ -49,64 +48,65 @@ def executable_suffix(self):
     def library_path_variable_name(self):
         return "LIB"
 
-    @staticmethod
-    def cmake_already_installed(required_version: str) -> bool:
-        cmake_path = shutil.which("cmake")
-        if not cmake_path:
-            return False
-
-        output = subprocess.check_output([cmake_path, "--version"])
-        cmake_version_output = six.ensure_str(output).splitlines()[0]
-        installed_version = cmake_version_output.replace("cmake version ", "")
-        return LooseVersion(installed_version) >= LooseVersion(required_version)
-
     @classmethod
-    def prepare_file(cls, deps_dir: str, zip_path: str, full_spec: str):
+    def download_and_extract_dependency(cls, zip_path: str, full_spec: str):
         if not os.path.isfile(zip_path):
-            zip_url = "{}{}.zip".format(DEPS_URL, urllib.parse.quote(full_spec))
+            zip_url = f"{DEPS_URL}{urllib.parse.quote(full_spec)}.zip"
             util.download_file(full_spec, zip_url, zip_path)
 
-        print("Extracting {}...".format(full_spec), end="")
+        zip_dir = os.path.dirname(zip_path)
+        print(f"Extracting {full_spec} to {zip_dir}...", end="")
         try:
-            util.extract(zip_path, deps_dir)
+            util.extract(zip_path, zip_dir)
         except zipfile.BadZipfile:
-            print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec)
+            print(f"\nError: {full_spec}.zip is not a valid zip file, redownload...")
             os.remove(zip_path)
-            cls.prepare_file(deps_dir, zip_path, full_spec)
+            cls.download_and_extract_dependency(zip_path, full_spec)
         else:
             print("done")
 
-    def _platform_bootstrap(self, cache_dir: str, _force: bool = False) -> bool:
-        deps_dir = os.path.join(cache_dir, "msvc-dependencies")
-
-        def get_package_dir(package, version) -> str:
-            return os.path.join(deps_dir, package, version)
-
-        to_install = {}
-        for package, version in DEPENDENCIES.items():
-            # Don't install CMake if it already exists in PATH
-            if package == "cmake" and self.cmake_already_installed(version):
-                continue
-
-            if not os.path.isdir(get_package_dir(package, version)):
-                to_install[package] = version
+    def _platform_bootstrap(self, force: bool = False) -> bool:
+        installed_something = self.passive_bootstrap()
 
+        try:
+            choco_config = os.path.join(util.SERVO_ROOT, "support", "windows", "chocolatey.config")
+
+            # This is the format that PowerShell wants arguments passed to it.
+            cmd_exe_args = f"'/K','choco','install','-y','{choco_config}'"
+            if force:
+                cmd_exe_args += ",'-f'"
+
+            print(cmd_exe_args)
+            subprocess.check_output([
+                "powershell", "Start-Process", "-Wait", "-verb", "runAs",
+                "cmd.exe", "-ArgumentList", f"@({cmd_exe_args})"
+            ]).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            print("Could not run chocolatey.  Follow manual build setup instructions.")
+            raise e
+
+        return installed_something
+
+    def passive_bootstrap(self) -> bool:
+        """A bootstrap method that is called without explicitly invoking `./mach bootstrap`
+           but that is executed in the process of other `./mach` commands. This should be
+           as fast as possible."""
+        to_install = [package for package in DEPENDENCIES if
+                      not os.path.isdir(get_dependency_dir(package))]
         if not to_install:
             return False
 
         print("Installing missing MSVC dependencies...")
-        for package, version in to_install.items():
-            full_spec = "{}-{}".format(package, version)
+        for package in to_install:
+            full_spec = "{}-{}".format(package, DEPENDENCIES[package])
 
-            package_dir = get_package_dir(package, version)
+            package_dir = get_dependency_dir(package)
             parent_dir = os.path.dirname(package_dir)
             if not os.path.isdir(parent_dir):
                 os.makedirs(parent_dir)
 
-            self.prepare_file(deps_dir, package_dir + ".zip", full_spec)
-
-            extracted_path = os.path.join(deps_dir, full_spec)
-            os.rename(extracted_path, package_dir)
+            self.download_and_extract_dependency(package_dir + ".zip", full_spec)
+            os.rename(os.path.join(parent_dir, full_spec), package_dir)
 
         return True
 

support/windows/chocolatey.config

@@ -1,8 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="cmake" version="3.26.4" />
+  <package id="cmake" version="3.26.4" installArguments="ADD_CMAKE_TO_PATH=System"/>
   <package id="git"/>
-  <package id="llvm"/>
   <package id="ninja"/>
   <package id="nuget.commandline" version="6.6.0" />
   <package id="python"/>