Use the rust-src tool for fetching rust source (#844)

This tool only includes the relevant rust source files, significantly
slimming the size of the download, and significantly decreasing the
number of bazel targets generated. The concept of a "host tool" is
introduced, which is targetless, to accommodate this change.

Author: csmulhern

rust/known_shas.bzl

@@ -316,11 +316,11 @@ def load_rust_src(ctx):
     Args:
         ctx (ctx): A repository_ctx.
     """
-    tool_suburl = produce_tool_suburl("rustc", "src", ctx.attr.version, ctx.attr.iso_date)
+    tool_suburl = produce_tool_suburl("rust-src", None, ctx.attr.version, ctx.attr.iso_date)
     static_rust = ctx.os.environ.get("STATIC_RUST_URL", "https://static.rust-lang.org")
     url = "{}/dist/{}.tar.gz".format(static_rust, tool_suburl)
 
-    tool_path = produce_tool_path("rustc", "src", ctx.attr.version)
+    tool_path = produce_tool_path("rust-src", None, ctx.attr.version)
     archive_path = tool_path + ".tar.gz"
     ctx.download(
         url,
@@ -330,7 +330,7 @@ def load_rust_src(ctx):
     ctx.extract(
         archive_path,
         output = "lib/rustlib/src",
-        stripPrefix = tool_path,
+        stripPrefix = "{}/rust-src/lib/rustlib/src/rust".format(tool_path),
     )
     ctx.file(
         "lib/rustlib/src/BUILD.bazel",
@@ -463,12 +463,12 @@ def produce_tool_suburl(tool_name, target_triple, version, iso_date = None):
         target_triple: The rust-style target triple of the tool
         version: The version of the tool among "nightly", "beta', or an exact version.
         iso_date: The date of the tool (or None, if the version is a specific version).
-    """
 
-    if iso_date:
-        return "{}/{}-{}-{}".format(iso_date, tool_name, version, target_triple)
-    else:
-        return "{}-{}-{}".format(tool_name, version, target_triple)
+    Returns:
+        str: The fully qualified url path for the specified tool.
+    """
+    path = produce_tool_path(tool_name, target_triple, version)
+    return iso_date + "/" + path if iso_date else path
 
 def produce_tool_path(tool_name, target_triple, version):
     """Produces a qualified Rust tool name
@@ -477,9 +477,15 @@ def produce_tool_path(tool_name, target_triple, version):
         tool_name: The name of the tool per static.rust-lang.org
         target_triple: The rust-style target triple of the tool
         version: The version of the tool among "nightly", "beta', or an exact version.
-    """
 
-    return "{}-{}-{}".format(tool_name, version, target_triple)
+    Returns:
+        str: The qualified path for the specified tool.
+    """
+    if not tool_name:
+        fail("No tool name was provided")
+    if not version:
+        fail("No tool version was provided")
+    return "-".join([e for e in [tool_name, version, target_triple] if e])
 
 def load_arbitrary_tool(ctx, tool_name, tool_subdirectories, version, iso_date, target_triple, sha256 = ""):
     """Loads a Rust tool, downloads, and extracts into the common workspace.

rust/private/repository_utils.bzl

@@ -26,6 +26,25 @@ def _produce_tool_suburl_test_impl(ctx):
             target_triple = "x86_64-unknown-linux-gnu",
         ),
     )
+    asserts.equals(
+        env,
+        "2020-05-22/rust-src-nightly",
+        produce_tool_suburl(
+            iso_date = "2020-05-22",
+            tool_name = "rust-src",
+            version = "nightly",
+            target_triple = None,
+        ),
+    )
+    asserts.equals(
+        env,
+        "rust-src-nightly",
+        produce_tool_suburl(
+            tool_name = "rust-src",
+            version = "nightly",
+            target_triple = None,
+        ),
+    )
     return unittest.end(env)
 
 def _produce_tool_path_test_impl(ctx):
@@ -39,6 +58,15 @@ def _produce_tool_path_test_impl(ctx):
             target_triple = "x86_64-unknown-linux-gnu",
         ),
     )
+    asserts.equals(
+        env,
+        "rust-src-nightly",
+        produce_tool_path(
+            tool_name = "rust-src",
+            version = "nightly",
+            target_triple = None,
+        ),
+    )
     return unittest.end(env)
 
 produce_tool_suburl_test = unittest.make(_produce_tool_suburl_test_impl)

test/unit/repository_utils/repository_utils_test.bzl

@@ -12,6 +12,7 @@ if [[ -z "${BUILD_WORKSPACE_DIRECTORY}" ]]; then
 fi
 
 TOOLS="$(cat "${BUILD_WORKSPACE_DIRECTORY}/util/fetch_shas_TOOLS.txt")"
+HOST_TOOLS="$(cat "${BUILD_WORKSPACE_DIRECTORY}/util/fetch_shas_HOST_TOOLS.txt")"
 TARGETS="$(cat "${BUILD_WORKSPACE_DIRECTORY}/util/fetch_shas_TARGETS.txt")"
 VERSIONS="$(cat "${BUILD_WORKSPACE_DIRECTORY}/util/fetch_shas_VERSIONS.txt")"
 BETA_ISO_DATES="$(cat "${BUILD_WORKSPACE_DIRECTORY}/util/fetch_shas_BETA_ISO_DATES.txt")"
@@ -39,6 +40,14 @@ enumerate_keys() {
       done
     done
   done
+
+  for HOST_TOOL in $HOST_TOOLS
+  do
+    for VERSION in $VERSIONS
+    do
+      echo "$HOST_TOOL-$VERSION"
+    done
+  done
 }
 
 emit_bzl_file_contents() {

util/fetch_shas.sh

@@ -0,0 +1 @@
+rust-src

util/fetch_shas_HOST_TOOLS.txt

@@ -1,7 +1,6 @@
 aarch64-apple-darwin
 aarch64-unknown-linux-gnu
 aarch64-unknown-linux-musl
-src
 wasm32-unknown-unknown
 wasm32-wasi
 x86_64-apple-darwin