update_rust.py will try the latest and fallback revisions

Have --print-package-version print the stamp version, as the Clang
update.py script does too. This tells the actual version of the
toolchain in the repo.

When updating Rust, check if the latest Rust/Clang revisions have a
Rust toolchain package. Use if it so. If not, try the
FALLBACK_REVISION.

After we have chosen a version, then proceed to check if the stamp
matches, and download a replacement toolchain if not.

This should mean that clang rolls no longer need to bump the
FALLBACK_REVISION to succeed. We can do it after the fact if we
find that we can't roll Rust, to pin it to something newish.

Bug: 1401042
Change-Id: Ic0436e87d18566f3498d30f3560fcc4e196ec240
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4406712
Reviewed-by: Collin Baker <collinbaker@chromium.org>
Commit-Queue: danakj <danakj@chromium.org>
Reviewed-by: Zequan Wu <zequanwu@google.com>
Cr-Commit-Position: refs/heads/main@{#1128845}
NOKEYCHECK=True
GitOrigin-RevId: 8f23682d65831ac89b32764d98238513d3950cc3

Author: danakj

build_rust.py

@@ -60,7 +60,7 @@
 
 from update_rust import (RUST_REVISION, RUST_TOOLCHAIN_OUT_DIR,
                          STAGE0_JSON_SHA256, THIRD_PARTY_DIR, THIS_DIR,
-                         VERSION_STAMP_PATH, GetPackageVersionForBuild)
+                         VERSION_STAMP_PATH, GetLatestRevision)
 
 EXCLUDED_TESTS = [
     # https://github.com/rust-lang/rust/issues/45222 which appears to have
@@ -442,7 +442,7 @@ def quote_string(s: str):
         subs = {}
         subs['INSTALL_DIR'] = quote_string(str(RUST_TOOLCHAIN_OUT_DIR))
         subs['LLVM_BIN'] = quote_string(str(self._llvm_bins_path))
-        subs['PACKAGE_VERSION'] = GetPackageVersionForBuild()
+        subs['PACKAGE_VERSION'] = GetLatestRevision()
 
         subs["LLVM_CONFIG_WINDOWS_x86_64"] = quote_string(
             str(x86_64_llvm_config))
@@ -490,7 +490,7 @@ def MakeVersionStamp(git_hash):
     with open(RUST_SRC_VERSION_FILE_PATH) as version_file:
         rust_version = version_file.readline().rstrip()
     return (f'rustc {rust_version} {git_hash}'
-            f' ({GetPackageVersionForBuild()} chromium)\n')
+            f' ({GetLatestRevision()} chromium)\n')
 
 
 def GetLatestRustCommit():

package_rust.py

@@ -15,11 +15,11 @@
     os.path.join(os.path.dirname(THIS_DIR), '..', 'clang', 'scripts'))
 
 from build_rust import (RUST_TOOLCHAIN_OUT_DIR, THIRD_PARTY_DIR)
-from update_rust import (GetPackageVersionForBuild)
+from update_rust import (GetLatestRevision)
 from package import (MaybeUpload, TeeCmd)
 from update import (CHROMIUM_DIR)
 
-PACKAGE_VERSION = GetPackageVersionForBuild()
+PACKAGE_VERSION = GetLatestRevision()
 BUILDLOG_NAME = f'rust-buildlog-{PACKAGE_VERSION}.txt'
 RUST_TOOLCHAIN_PACKAGE_NAME = f'rust-toolchain-{PACKAGE_VERSION}.tgz'
 

update_rust.py

@@ -16,7 +16,7 @@
 import re
 import shutil
 import sys
-import tempfile
+import time
 import urllib
 
 from pathlib import Path
@@ -82,20 +82,20 @@
 # Package version built in build_rust.py, which is always built against the
 # current Clang. Typically Clang and Rust revisions are both updated together
 # and this picks the Clang that has just been built.
-def GetPackageVersionForBuild():
+def GetLatestRevision():
     from update import (CLANG_REVISION, CLANG_SUB_REVISION)
     return (f'{RUST_REVISION}-{RUST_SUB_REVISION}'
             f'-{CLANG_REVISION}-{CLANG_SUB_REVISION}')
 
 
 # Package version for download. Ideally this is the latest Clang+Rust roll,
-# which was built successfully and is returned from GetPackageVersionForBuild().
+# which was built successfully and is returned from GetLatestRevision().
 # However at this time Clang rolls even if Rust fails to build, so we have Rust
 # pinned to the last known successful build with FALLBACK_REVISION. This should
 # go away once we block Clang rolls on Rust also being built.
 def GetDownloadPackageVersion():
     return FALLBACK_REVISION \
-        if FALLBACK_REVISION else GetPackageVersionForBuild()
+        if FALLBACK_REVISION else GetLatestRevision()
 
 
 # Get the version of the toolchain package we already have.
@@ -112,6 +112,28 @@ def GetStampVersion():
     return None
 
 
+def CheckUrl(url) -> bool:
+    """Check if a url exists."""
+    num_retries = 3
+    retry_wait_s = 5  # Doubled at each retry.
+
+    while True:
+        try:
+            urllib.request.urlopen(urllib.request.Request(url))
+            return True
+        except urllib.error.URLError as e:
+            if e.code != 404:
+                print(e)
+            if num_retries == 0 or isinstance(
+                    e, urllib.error.HTTPError) and e.code == 404:
+                return False
+            num_retries -= 1
+            print(f'Retrying in {retry_wait_s} s ...')
+            sys.stdout.flush()
+            time.sleep(retry_wait_s)
+            retry_wait_s *= 2
+
+
 def main():
     parser = argparse.ArgumentParser(description='Update Rust package')
     parser.add_argument(
@@ -130,39 +152,54 @@ def main():
         return 0
 
     if args.print_package_version:
-        print(GetDownloadPackageVersion())
+        stamp_version = GetStampVersion()
+        if (stamp_version != GetLatestRevision()
+                and stamp_version != FALLBACK_REVISION):
+            print(
+                f'The expected Rust version is {GetLatestRevision()} '
+                f'(or fallback {FALLBACK_REVISION} but the actual version is '
+                f'{stamp_version}')
+            print('Did you run "gclient sync"?')
+            return 1
+        print(stamp_version)
         return 0
 
     from update import (DownloadAndUnpack, GetDefaultHostOs,
                         GetPlatformUrlPrefix)
 
+    platform_prefix = GetPlatformUrlPrefix(GetDefaultHostOs())
+
+    version = GetLatestRevision()
+    url = f'{platform_prefix}rust-toolchain-{version}.tgz'
+    if not CheckUrl(url):
+        print("Latest Rust toolchain not found. Using fallback revision.")
+        version = FALLBACK_REVISION
+        url = f'{platform_prefix}rust-toolchain-{version}.tgz'
+        if not CheckUrl(url):
+            print('error: Could not find Rust toolchain package')
+            return 1
+
     # Exit early if the existing package is up-to-date. Note that we cannot
     # simply call DownloadAndUnpack() every time: aside from unnecessarily
     # downloading the toolchain if it hasn't changed, it also leads to multiple
     # versions of the same rustlibs. build/rust/std/find_std_rlibs.py chokes in
     # this case.
     if os.path.exists(RUST_TOOLCHAIN_OUT_DIR):
-        if GetDownloadPackageVersion() == GetStampVersion():
+        if version == GetStampVersion():
             return 0
 
     if os.path.exists(RUST_TOOLCHAIN_OUT_DIR):
         shutil.rmtree(RUST_TOOLCHAIN_OUT_DIR)
 
     try:
         platform_prefix = GetPlatformUrlPrefix(GetDefaultHostOs())
-        version = GetDownloadPackageVersion()
-        url = f'{platform_prefix}rust-toolchain-{version}.tgz'
         DownloadAndUnpack(url, THIRD_PARTY_DIR)
     except urllib.error.HTTPError as e:
-        # Fail softly for now. This can happen if a Rust package was not
-        # produced, e.g. if the Rust build failed upon a Clang update, or if a
-        # Rust roll and a Clang roll raced against each other.
-        #
-        # TODO(https://crbug.com/1245714): Reconsider how to handle this.
-        print(f'warning: could not download Rust package')
+        print(f'error: Failed to download Rust package')
+        return 1
 
     # Ensure the newly extracted package has the correct version.
-    assert GetDownloadPackageVersion() == GetStampVersion()
+    assert version == GetStampVersion()
 
 
 if __name__ == '__main__':