create target-specs folder per installation, pulled from rustc_codegen_spirv_target_specs, with legacy fallback

Author: Firestar99

Cargo.lock

@@ -13,7 +13,7 @@ exclude = [
 resolver = "2"
 
 [workspace.dependencies]
-spirv-builder = { git = "https://github.com/Rust-GPU/rust-gpu.git", rev = "6d7c1cd6c0920500a3fa8c01c23e7b74302c15c4", default-features = false }
+spirv-builder = { git = "https://github.com/Rust-GPU/rust-gpu.git", rev = "9b99f208414473ff39afacbf91b0c29f3bd6b3e3", default-features = false }
 anyhow = "1.0.94"
 clap = { version = "4.5.37", features = ["derive"] }
 crossterm = "0.28.1"
@@ -29,6 +29,9 @@ test-log = "0.2.16"
 cargo_metadata = "0.19.2"
 semver = "1.0.26"
 
+# This crate MUST NEVER be upgraded, we need this particular "first" version to support old rust-gpu builds
+legacy_target_specs = { git = "https://github.com/Rust-GPU/rust-gpu.git", rev = "9b99f208414473ff39afacbf91b0c29f3bd6b3e3", package = "rustc_codegen_spirv-target-specs", features = ["include_str"] }
+
 [workspace.lints.rust]
 missing_docs = "warn"
 

Cargo.toml

@@ -14,6 +14,7 @@ default-run = "cargo-gpu"
 cargo_metadata.workspace = true
 anyhow.workspace = true
 spirv-builder = { workspace = true, features = ["clap", "watch"] }
+legacy_target_specs.workspace = true
 clap.workspace = true
 directories.workspace = true
 env_logger.workspace = true

crates/cargo-gpu/Cargo.toml

@@ -1,13 +1,14 @@
 //! Install a dedicated per-shader crate that has the `rust-gpu` compiler in it.
 
+use crate::legacy_target_specs::write_legacy_target_specs;
 use crate::spirv_source::{
     get_channel_from_rustc_codegen_spirv_build_script, query_metadata, FindPackage as _,
 };
-use crate::{cache_dir, spirv_source::SpirvSource, target_spec_dir};
+use crate::{cache_dir, spirv_source::SpirvSource};
 use anyhow::Context as _;
-use log::trace;
-use spirv_builder::{SpirvBuilder, TARGET_SPECS};
-use std::io::Write as _;
+use cargo_metadata::Metadata;
+use log::{info, trace};
+use spirv_builder::SpirvBuilder;
 use std::path::{Path, PathBuf};
 
 /// Args for an install
@@ -83,14 +84,16 @@ pub struct InstalledBackend {
     pub rustc_codegen_spirv_location: PathBuf,
     /// toolchain channel name
     pub toolchain_channel: String,
+    /// directory with target-specs json files
+    pub target_spec_dir: PathBuf,
 }
 
 impl InstalledBackend {
     /// Configures the supplied [`SpirvBuilder`]. `SpirvBuilder.target` must be set and must not change after calling this function.
     pub fn configure_spirv_builder(&self, builder: &mut SpirvBuilder) -> anyhow::Result<()> {
         builder.rustc_codegen_spirv_location = Some(self.rustc_codegen_spirv_location.clone());
         builder.toolchain_overwrite = Some(self.toolchain_channel.clone());
-        builder.path_to_target_spec = Some(target_spec_dir()?.join(format!(
+        builder.path_to_target_spec = Some(self.target_spec_dir.join(format!(
             "{}.json",
             builder.target.as_ref().context("expect target to be set")?
         )));
@@ -169,22 +172,72 @@ package = "rustc_codegen_spirv"
         Ok(())
     }
 
-    /// Add the target spec files to the crate.
-    fn write_target_spec_files(&self) -> anyhow::Result<()> {
-        for (filename, contents) in TARGET_SPECS {
-            let path = target_spec_dir()
-                .context("creating target spec dir")?
-                .join(filename);
-            if !path.is_file() || self.rebuild_codegen {
-                let mut file = std::fs::File::create(&path)
-                    .with_context(|| format!("creating file at [{}]", path.display()))?;
-                file.write_all(contents.as_bytes())
-                    .context("writing to file")?;
+    /// Copy spec files from one dir to another, assuming no subdirectories
+    fn copy_spec_files(src: &Path, dst: &Path) -> anyhow::Result<()> {
+        info!(
+            "Copy target specs from {:?} to {:?}",
+            src.display(),
+            dst.display()
+        );
+        std::fs::create_dir_all(dst)?;
+        let dir = std::fs::read_dir(src)?;
+        for dir_entry in dir {
+            let file = dir_entry?;
+            let file_path = file.path();
+            if file_path.is_file() {
+                std::fs::copy(file_path, dst.join(file.file_name()))?;
             }
         }
         Ok(())
     }
 
+    /// Add the target spec files to the crate.
+    fn update_spec_files(
+        &self,
+        source: &SpirvSource,
+        install_dir: &Path,
+        dummy_metadata: &Metadata,
+        skip_rebuild: bool,
+    ) -> anyhow::Result<PathBuf> {
+        let mut target_specs_dst = install_dir.join("target-specs");
+        if !skip_rebuild {
+            if let Ok(target_specs) =
+                dummy_metadata.find_package("rustc_codegen_spirv-target-specs")
+            {
+                let target_specs_src = target_specs
+                    .manifest_path
+                    .as_std_path()
+                    .parent()
+                    .and_then(|p| {
+                        let src = p.join("target-specs");
+                        src.is_dir().then_some(src)
+                    })
+                    .context("Could not find `target-specs` directory within `rustc_codegen_spirv-target-specs` dependency")?;
+                if source.is_path() {
+                    // skip copy
+                    target_specs_dst = target_specs_src;
+                } else {
+                    // copy over the target-specs
+                    Self::copy_spec_files(&target_specs_src, &target_specs_dst)
+                        .context("copying target-specs json files")?;
+                }
+            } else {
+                // use legacy target specs bundled with cargo gpu
+                if source.is_path() {
+                    // This is a stupid situation:
+                    // * We can't be certain that there are `target-specs` in the local checkout (there may be some in `spirv-builder`)
+                    // * We can't dump our legacy ones into the `install_dir`, as that would modify the local rust-gpu checkout
+                    // -> do what the old cargo gpu did, one global dir for all target specs
+                    // and hope parallel runs don't shred each other
+                    target_specs_dst = cache_dir()?.join("legacy-target-specs-for-local-checkout");
+                }
+                write_legacy_target_specs(&target_specs_dst, self.rebuild_codegen)?;
+            }
+        }
+
+        Ok(target_specs_dst)
+    }
+
     /// Install the binary pair and return the `(dylib_path, toolchain_channel)`.
     #[expect(clippy::too_many_lines, reason = "it's fine")]
     pub fn run(&self) -> anyhow::Result<InstalledBackend> {
@@ -245,6 +298,11 @@ package = "rustc_codegen_spirv"
             )?;
         log::info!("selected toolchain channel `{toolchain_channel:?}`");
 
+        log::debug!("update_spec_files");
+        let target_spec_dir = self
+            .update_spec_files(&source, &install_dir, &dummy_metadata, skip_rebuild)
+            .context("writing target spec files")?;
+
         if !skip_rebuild {
             log::debug!("ensure_toolchain_and_components_exist");
             crate::install_toolchain::ensure_toolchain_and_components_exist(
@@ -304,15 +362,12 @@ package = "rustc_codegen_spirv"
                 log::error!("could not find {}", dylib_path.display());
                 anyhow::bail!("`rustc_codegen_spirv` build failed");
             }
-
-            log::debug!("write_target_spec_files");
-            self.write_target_spec_files()
-                .context("writing target spec files")?;
         }
 
         Ok(InstalledBackend {
             rustc_codegen_spirv_location: dest_dylib_path,
             toolchain_channel,
+            target_spec_dir,
         })
     }
 }

crates/cargo-gpu/src/install.rs

@@ -0,0 +1,26 @@
+//! Legacy target specs are spec jsons for versions before `rustc_codegen_spirv-target-specs`
+//! came bundled with them. Instead, cargo gpu needs to bundle these legacy spec files. Luckily,
+//! they are the same for all versions, as bundling target specs with the codegen backend was
+//! introduced before the first target spec update.
+
+use anyhow::Context as _;
+use log::info;
+use std::path::Path;
+
+/// Extract legacy target specs from our executable into some directory
+pub fn write_legacy_target_specs(target_spec_dir: &Path, rebuild: bool) -> anyhow::Result<()> {
+    info!(
+        "Writing legacy target specs to {}",
+        target_spec_dir.display()
+    );
+    std::fs::create_dir_all(target_spec_dir)?;
+    for (filename, contents) in legacy_target_specs::TARGET_SPECS {
+        let path = target_spec_dir.join(filename);
+        if !path.is_file() || rebuild {
+            std::fs::write(&path, contents.as_bytes()).with_context(|| {
+                format!("writing legacy target spec file at [{}]", path.display())
+            })?;
+        }
+    }
+    Ok(())
+}

crates/cargo-gpu/src/legacy_target_specs.rs

@@ -59,6 +59,7 @@ mod build;
 mod config;
 mod install;
 mod install_toolchain;
+mod legacy_target_specs;
 mod linkage;
 mod lockfile;
 mod metadata;
@@ -201,13 +202,6 @@ fn cache_dir() -> anyhow::Result<std::path::PathBuf> {
     })
 }
 
-/// Location of the target spec metadata files
-fn target_spec_dir() -> anyhow::Result<std::path::PathBuf> {
-    let dir = cache_dir()?.join("target-specs");
-    std::fs::create_dir_all(&dir)?;
-    Ok(dir)
-}
-
 /// Convenience function for internal use. Dumps all the CLI usage instructions. Useful for
 /// updating the README.
 fn dump_full_usage_for_readme() -> anyhow::Result<()> {

crates/cargo-gpu/src/main.rs

@@ -253,7 +253,7 @@ mod test {
             source,
             SpirvSource::Git {
                 url: "https://github.com/Rust-GPU/rust-gpu".to_owned(),
-                rev: "82a0f69008414f51d59184763146caa6850ac588".to_owned()
+                rev: "86edfffac1d2fe349211576617ffc22a176de2a3".to_owned()
             }
         );
     }
@@ -275,6 +275,6 @@ mod test {
             .to_str()
             .map(std::string::ToString::to_string)
             .unwrap();
-        assert_eq!("https___github_com_Rust-GPU_rust-gpu+82a0f690", &name);
+        assert_eq!("https___github_com_Rust-GPU_rust-gpu+86edfffa", &name);
     }
 }

crates/cargo-gpu/src/spirv_source.rs

@@ -9,7 +9,7 @@ crate-type = ["rlib", "cdylib"]
 # Dependencies for CPU and GPU code
 [dependencies]
 # TODO: use a simple crate version once v0.10.0 is released
-spirv-std = { git = "https://github.com/Rust-GPU/rust-gpu", rev = "82a0f69" }
+spirv-std = { git = "https://github.com/Rust-GPU/rust-gpu", rev = "86edfffac1d2fe349211576617ffc22a176de2a3" }
 
 # Dependencies for GPU code
 [target.'cfg(target_arch = "spirv")'.dependencies]