diff --git a/Cargo.lock b/Cargo.lock index 992488f..1714216 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,6 +109,7 @@ dependencies = [ "env_logger 0.10.2", "log", "relative-path", + "rustc_codegen_spirv-target-specs", "semver", "serde", "serde_json", @@ -744,10 +745,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_codegen_spirv-target-specs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c89eaf493b3dfc730cda42a77014aad65e03213992c7afe0dff60a9f7d3dd94" + [[package]] name = "rustc_codegen_spirv-types" version = "0.9.0" -source = "git+https://github.com/Rust-GPU/rust-gpu.git?rev=6d7c1cd6c0920500a3fa8c01c23e7b74302c15c4#6d7c1cd6c0920500a3fa8c01c23e7b74302c15c4" +source = "git+https://github.com/Rust-GPU/rust-gpu.git?rev=86fc48032c4cd4afb74f1d81ae859711d20386a1#86fc48032c4cd4afb74f1d81ae859711d20386a1" dependencies = [ "rspirv", "serde", @@ -897,7 +904,7 @@ dependencies = [ [[package]] name = "spirv-builder" version = "0.9.0" -source = "git+https://github.com/Rust-GPU/rust-gpu.git?rev=6d7c1cd6c0920500a3fa8c01c23e7b74302c15c4#6d7c1cd6c0920500a3fa8c01c23e7b74302c15c4" +source = "git+https://github.com/Rust-GPU/rust-gpu.git?rev=86fc48032c4cd4afb74f1d81ae859711d20386a1#86fc48032c4cd4afb74f1d81ae859711d20386a1" dependencies = [ "clap", "memchr", diff --git a/Cargo.toml b/Cargo.toml index c266e87..036c971 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ exclude = [ resolver = "2" [workspace.dependencies] -spirv-builder = { git = "https://github.com/Rust-GPU/rust-gpu.git", rev = "6d7c1cd6c0920500a3fa8c01c23e7b74302c15c4", default-features = false } +spirv-builder = { git = "https://github.com/Rust-GPU/rust-gpu.git", rev = "86fc48032c4cd4afb74f1d81ae859711d20386a1", default-features = false } anyhow = "1.0.94" clap = { version = "4.5.37", features = ["derive"] } crossterm = "0.28.1" @@ -29,6 +29,9 @@ test-log = "0.2.16" cargo_metadata = "0.19.2" semver = "1.0.26" +# This crate MUST NEVER be upgraded, we need this particular "first" version to support old rust-gpu builds +legacy_target_specs = { package = "rustc_codegen_spirv-target-specs", version = "0.9.0", features = ["include_str"] } + [workspace.lints.rust] missing_docs = "warn" diff --git a/crates/cargo-gpu/Cargo.toml b/crates/cargo-gpu/Cargo.toml index 49ea815..9152876 100644 --- a/crates/cargo-gpu/Cargo.toml +++ b/crates/cargo-gpu/Cargo.toml @@ -14,6 +14,7 @@ default-run = "cargo-gpu" cargo_metadata.workspace = true anyhow.workspace = true spirv-builder = { workspace = true, features = ["clap", "watch"] } +legacy_target_specs.workspace = true clap.workspace = true directories.workspace = true env_logger.workspace = true diff --git a/crates/cargo-gpu/src/args.rs b/crates/cargo-gpu/src/args.rs deleted file mode 100644 index 346d857..0000000 --- a/crates/cargo-gpu/src/args.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! Args for building and installing. - -use spirv_builder::SpirvBuilder; - -/// All args for a build and install -#[derive(clap::Parser, Debug, Clone, serde::Deserialize, serde::Serialize)] -pub struct AllArgs { - /// build args - #[clap(flatten)] - pub build: BuildArgs, - - /// install args - #[clap(flatten)] - pub install: InstallArgs, -} - -/// Args for just a build -#[derive(clap::Parser, Debug, Clone, serde::Deserialize, serde::Serialize)] -pub struct BuildArgs { - /// Path to the output directory for the compiled shaders. - #[clap(long, short, default_value = "./")] - pub output_dir: std::path::PathBuf, - - /// Watch the shader crate directory and automatically recompile on changes. - #[clap(long, short, action)] - pub watch: bool, - - /// the flattened [`SpirvBuilder`] - #[clap(flatten)] - #[serde(flatten)] - pub spirv_builder: SpirvBuilder, - - ///Renames the manifest.json file to the given name - #[clap(long, short, default_value = "manifest.json")] - pub manifest_file: String, -} - -/// Args for an install -#[derive(clap::Parser, Debug, Clone, serde::Deserialize, serde::Serialize)] -#[expect( - clippy::struct_excessive_bools, - reason = "cmdline args have many bools" -)] -pub struct InstallArgs { - /// path to the `rustc_codegen_spirv` dylib - #[clap(long, hide(true), default_value = "INTERNALLY_SET")] - pub dylib_path: std::path::PathBuf, - - /// Directory containing the shader crate to compile. - #[clap(long, default_value = "./")] - pub shader_crate: std::path::PathBuf, - - #[expect( - clippy::doc_markdown, - reason = "The URL should appear literally like this. But Clippy wants a markdown clickable link" - )] - /// Source of `spirv-builder` dependency - /// Eg: "https://github.com/Rust-GPU/rust-gpu" - #[clap(long)] - pub spirv_builder_source: Option, - - /// Version of `spirv-builder` dependency. - /// * If `--spirv-builder-source` is not set, then this is assumed to be a crates.io semantic - /// version such as "0.9.0". - /// * If `--spirv-builder-source` is set, then this is assumed to be a Git "commitsh", such - /// as a Git commit hash or a Git tag, therefore anything that `git checkout` can resolve. - #[clap(long, verbatim_doc_comment)] - pub spirv_builder_version: Option, - - /// Force `rustc_codegen_spirv` to be rebuilt. - #[clap(long)] - pub rebuild_codegen: bool, - - /// Assume "yes" to "Install Rust toolchain: [y/n]" prompt. - #[clap(long, action)] - pub auto_install_rust_toolchain: bool, - - /// Clear target dir of `rustc_codegen_spirv` build after a successful build, saves about - /// 200MiB of disk space. - #[clap(long = "no-clear-target", default_value = "true", action = clap::ArgAction::SetFalse)] - pub clear_target: bool, - - /// There is a tricky situation where a shader crate that depends on workspace config can have - /// a different `Cargo.lock` lockfile version from the the workspace's `Cargo.lock`. This can - /// prevent builds when an old Rust toolchain doesn't recognise the newer lockfile version. - /// - /// The ideal way to resolve this would be to match the shader crate's toolchain with the - /// workspace's toolchain. However, that is not always possible. Another solution is to - /// `exclude = [...]` the problematic shader crate from the workspace. This also may not be a - /// suitable solution if there are a number of shader crates all sharing similar config and - /// you don't want to have to copy/paste and maintain that config across all the shaders. - /// - /// So a somewhat hacky workaround is to have `cargo gpu` overwrite lockfile versions. Enabling - /// this flag will only come into effect if there are a mix of v3/v4 lockfiles. It will also - /// only overwrite versions for the duration of a build. It will attempt to return the versions - /// to their original values once the build is finished. However, of course, unexpected errors - /// can occur and the overwritten values can remain. Hence why this behaviour is not enabled by - /// default. - /// - /// This hack is possible because the change from v3 to v4 only involves a minor change to the - /// way source URLs are encoded. See these PRs for more details: - /// * - /// * - #[clap(long, action, verbatim_doc_comment)] - pub force_overwrite_lockfiles_v4_to_v3: bool, -} diff --git a/crates/cargo-gpu/src/build.rs b/crates/cargo-gpu/src/build.rs index b77e5a2..1bb5078 100644 --- a/crates/cargo-gpu/src/build.rs +++ b/crates/cargo-gpu/src/build.rs @@ -2,13 +2,46 @@ #![allow(clippy::unwrap_used, reason = "this is basically a test")] //! `cargo gpu build`, analogous to `cargo build` -use crate::args::BuildArgs; +use crate::install::Install; use crate::linkage::Linkage; use crate::lockfile::LockfileMismatchHandler; -use crate::{install::Install, target_spec_dir}; use anyhow::Context as _; -use spirv_builder::{CompileResult, ModuleResult}; +use spirv_builder::{CompileResult, ModuleResult, SpirvBuilder}; use std::io::Write as _; +use std::path::PathBuf; + +/// Args for just a build +#[derive(clap::Parser, Debug, Clone, serde::Deserialize, serde::Serialize)] +pub struct BuildArgs { + /// Path to the output directory for the compiled shaders. + #[clap(long, short, default_value = "./")] + pub output_dir: PathBuf, + + /// Watch the shader crate directory and automatically recompile on changes. + #[clap(long, short, action)] + pub watch: bool, + + /// the flattened [`SpirvBuilder`] + #[clap(flatten)] + #[serde(flatten)] + pub spirv_builder: SpirvBuilder, + + ///Renames the manifest.json file to the given name + #[clap(long, short, default_value = "manifest.json")] + pub manifest_file: String, +} + +impl Default for BuildArgs { + #[inline] + fn default() -> Self { + Self { + output_dir: PathBuf::from("./"), + watch: false, + spirv_builder: SpirvBuilder::default(), + manifest_file: String::from("manifest.json"), + } + } +} /// `cargo build` subcommands #[derive(Clone, clap::Parser, Debug, serde::Deserialize, serde::Serialize)] @@ -19,54 +52,46 @@ pub struct Build { /// CLI args for configuring the build of the shader #[clap(flatten)] - pub build_args: BuildArgs, + pub build: BuildArgs, } impl Build { /// Entrypoint pub fn run(&mut self) -> anyhow::Result<()> { - let (rustc_codegen_spirv_location, toolchain_channel) = self.install.run()?; + let installed_backend = self.install.run()?; let _lockfile_mismatch_handler = LockfileMismatchHandler::new( - &self.install.spirv_install.shader_crate, - &toolchain_channel, - self.install - .spirv_install - .force_overwrite_lockfiles_v4_to_v3, + &self.install.shader_crate, + &installed_backend.toolchain_channel, + self.install.force_overwrite_lockfiles_v4_to_v3, )?; - let builder = &mut self.build_args.spirv_builder; - builder.rustc_codegen_spirv_location = Some(rustc_codegen_spirv_location); - builder.toolchain_overwrite = Some(toolchain_channel); - builder.path_to_crate = Some(self.install.spirv_install.shader_crate.clone()); - builder.path_to_target_spec = Some(target_spec_dir()?.join(format!( - "{}.json", - builder.target.as_ref().context("expect target to be set")? - ))); + let builder = &mut self.build.spirv_builder; + builder.path_to_crate = Some(self.install.shader_crate.clone()); + installed_backend.configure_spirv_builder(builder)?; // Ensure the shader output dir exists log::debug!( "ensuring output-dir '{}' exists", - self.build_args.output_dir.display() + self.build.output_dir.display() ); - std::fs::create_dir_all(&self.build_args.output_dir)?; - let canonicalized = self.build_args.output_dir.canonicalize()?; - log::debug!("canonicalized output dir: {canonicalized:?}"); - self.build_args.output_dir = canonicalized; + std::fs::create_dir_all(&self.build.output_dir)?; + let canonicalized = self.build.output_dir.canonicalize()?; + log::debug!("canonicalized output dir: {}", canonicalized.display()); + self.build.output_dir = canonicalized; // Ensure the shader crate exists - self.install.spirv_install.shader_crate = - self.install.spirv_install.shader_crate.canonicalize()?; + self.install.shader_crate = self.install.shader_crate.canonicalize()?; anyhow::ensure!( - self.install.spirv_install.shader_crate.exists(), + self.install.shader_crate.exists(), "shader crate '{}' does not exist. (Current dir is '{}')", - self.install.spirv_install.shader_crate.display(), + self.install.shader_crate.display(), std::env::current_dir()?.display() ); - if self.build_args.watch { + if self.build.watch { let this = self.clone(); - self.build_args + self.build .spirv_builder .watch(move |result, accept| { let result1 = this.parse_compilation_result(&result); @@ -79,9 +104,9 @@ impl Build { } else { crate::user_output!( "Compiling shaders at {}...\n", - self.install.spirv_install.shader_crate.display() + self.install.shader_crate.display() ); - let result = self.build_args.spirv_builder.build()?; + let result = self.build.spirv_builder.build()?; self.parse_compilation_result(&result)?; } Ok(()) @@ -104,7 +129,7 @@ impl Build { .into_iter() .map(|(entry, filepath)| -> anyhow::Result { use relative_path::PathExt as _; - let path = self.build_args.output_dir.join( + let path = self.build.output_dir.join( filepath .file_name() .context("Couldn't parse file name from shader module path")?, @@ -114,10 +139,10 @@ impl Build { log::debug!( "linkage of {} relative to {}", path.display(), - self.install.spirv_install.shader_crate.display() + self.install.shader_crate.display() ); let spv_path = path - .relative_to(&self.install.spirv_install.shader_crate) + .relative_to(&self.install.shader_crate) .map_or(path, |path_relative_to_shader_crate| { path_relative_to_shader_crate.to_path("") }); @@ -128,10 +153,7 @@ impl Build { linkage.sort(); // Write the shader manifest json file - let manifest_path = self - .build_args - .output_dir - .join(&self.build_args.manifest_file); + let manifest_path = self.build.output_dir.join(&self.build.manifest_file); let json = serde_json::to_string_pretty(&linkage)?; let mut file = std::fs::File::create(&manifest_path).with_context(|| { format!( @@ -176,8 +198,8 @@ mod test { command: Command::Build(build), } = Cli::parse_from(args) { - assert_eq!(shader_crate_path, build.install.spirv_install.shader_crate); - assert_eq!(output_dir, build.build_args.output_dir); + assert_eq!(shader_crate_path, build.install.shader_crate); + assert_eq!(output_dir, build.build.output_dir); // TODO: // For some reason running a full build (`build.run()`) inside tests fails on Windows. diff --git a/crates/cargo-gpu/src/config.rs b/crates/cargo-gpu/src/config.rs index ad86402..f1a67fa 100644 --- a/crates/cargo-gpu/src/config.rs +++ b/crates/cargo-gpu/src/config.rs @@ -14,28 +14,9 @@ impl Config { /// Convert CLI args to their serde JSON representation. fn cli_args_to_json(env_args: Vec) -> anyhow::Result { - let mut cli_args_json = serde_json::to_value(crate::build::Build::parse_from(env_args))?; - - // Move `/install/spirv_install` to `/install` - let spirv_install = cli_args_json - .pointer("/install/spirv_install") - .context("`/install/spirv_install` not found in config")? - .clone(); - *cli_args_json - .get_mut("install") - .context("`/install` not found in config")? = spirv_install; - - let build = cli_args_json - .pointer("/build_args") - .context("`/build_args` not found in config")? - .clone(); - - // Move `/build_args` to `/build` - let object = cli_args_json.as_object_mut().context("!")?; - object.remove("build_args"); - object.insert("build".to_owned(), build); - - Ok(cli_args_json) + Ok(serde_json::to_value(crate::build::Build::parse_from( + env_args, + ))?) } /// Config for the `cargo gpu build` and `cargo gpu install` can be set in the shader crate's @@ -47,30 +28,11 @@ impl Config { ) -> anyhow::Result { let mut config = crate::metadata::Metadata::as_json(shader_crate_path)?; - env_args = env_args - .into_iter() - .filter(|arg| !(arg == "build" || arg == "install")) - .collect::>(); + env_args.retain(|arg| !(arg == "build" || arg == "install")); let cli_args_json = Self::cli_args_to_json(env_args)?; - Self::json_merge(&mut config, cli_args_json, None)?; - let build = config - .get("build") - .context("`build` not found in merged configs")? - .clone(); - - let install = config - .get("install") - .context("`install` not found in merged configs")? - .clone(); - - let args = serde_json::from_value::(serde_json::json!({ - "build_args": build, - "install": { - "spirv_install": install - } - }))?; + let args = serde_json::from_value::(config)?; Ok(args) } @@ -140,8 +102,8 @@ mod test { ], ) .unwrap(); - assert!(!args.build_args.spirv_builder.release); - assert!(args.install.spirv_install.auto_install_rust_toolchain); + assert!(!args.build.spirv_builder.release); + assert!(args.install.auto_install_rust_toolchain); } #[test_log::test] @@ -161,8 +123,8 @@ mod test { .unwrap(); let args = Config::clap_command_with_cargo_config(&shader_crate_path, vec![]).unwrap(); - assert!(!args.build_args.spirv_builder.release); - assert!(args.install.spirv_install.auto_install_rust_toolchain); + assert!(!args.build.spirv_builder.release); + assert!(args.install.auto_install_rust_toolchain); } fn update_cargo_output_dir() -> std::path::PathBuf { @@ -186,15 +148,9 @@ mod test { let args = Config::clap_command_with_cargo_config(&shader_crate_path, vec![]).unwrap(); if cfg!(target_os = "windows") { - assert_eq!( - args.build_args.output_dir, - std::path::Path::new("C:/the/moon") - ); + assert_eq!(args.build.output_dir, std::path::Path::new("C:/the/moon")); } else { - assert_eq!( - args.build_args.output_dir, - std::path::Path::new("/the/moon") - ); + assert_eq!(args.build.output_dir, std::path::Path::new("/the/moon")); } } @@ -212,10 +168,7 @@ mod test { ], ) .unwrap(); - assert_eq!( - args.build_args.output_dir, - std::path::Path::new("/the/river") - ); + assert_eq!(args.build.output_dir, std::path::Path::new("/the/river")); } #[test_log::test] @@ -234,7 +187,7 @@ mod test { let args = Config::clap_command_with_cargo_config(&shader_crate_path, vec![]).unwrap(); assert_eq!( - args.build_args.spirv_builder.capabilities, + args.build.spirv_builder.capabilities, vec![ spirv_builder::Capability::AtomicStorage, spirv_builder::Capability::Matrix @@ -256,6 +209,6 @@ mod test { ], ) .unwrap(); - assert_eq!(args.build_args.manifest_file, "mymanifest".to_owned()); + assert_eq!(args.build.manifest_file, "mymanifest".to_owned()); } } diff --git a/crates/cargo-gpu/src/install.rs b/crates/cargo-gpu/src/install.rs index 0b6f5fd..bd08e6d 100644 --- a/crates/cargo-gpu/src/install.rs +++ b/crates/cargo-gpu/src/install.rs @@ -1,29 +1,126 @@ //! Install a dedicated per-shader crate that has the `rust-gpu` compiler in it. -use crate::args::InstallArgs; +use crate::legacy_target_specs::write_legacy_target_specs; use crate::spirv_source::{ - get_channel_from_rustc_codegen_spirv_build_script, get_package_from_crate, + get_channel_from_rustc_codegen_spirv_build_script, query_metadata, FindPackage as _, }; -use crate::{cache_dir, spirv_source::SpirvSource, target_spec_dir}; +use crate::{cache_dir, spirv_source::SpirvSource}; use anyhow::Context as _; -use log::trace; -use spirv_builder::TARGET_SPECS; -use std::io::Write as _; +use cargo_metadata::Metadata; +use log::{info, trace}; +use spirv_builder::SpirvBuilder; use std::path::{Path, PathBuf}; -/// `cargo gpu install` -#[derive(Clone, clap::Parser, Debug, serde::Deserialize, serde::Serialize)] +/// Args for an install +#[expect( + clippy::struct_excessive_bools, + reason = "cmdline args have many bools" +)] +#[derive(clap::Parser, Debug, Clone, serde::Deserialize, serde::Serialize)] pub struct Install { - /// CLI arguments for installing the Rust toolchain and components - #[clap(flatten)] - pub spirv_install: InstallArgs, + /// Directory containing the shader crate to compile. + #[clap(long, default_value = "./")] + pub shader_crate: PathBuf, + + #[expect( + clippy::doc_markdown, + reason = "The URL should appear literally like this. But Clippy wants a markdown clickable link" + )] + /// Source of `spirv-builder` dependency + /// Eg: "https://github.com/Rust-GPU/rust-gpu" + #[clap(long)] + pub spirv_builder_source: Option, + + /// Version of `spirv-builder` dependency. + /// * If `--spirv-builder-source` is not set, then this is assumed to be a crates.io semantic + /// version such as "0.9.0". + /// * If `--spirv-builder-source` is set, then this is assumed to be a Git "commitsh", such + /// as a Git commit hash or a Git tag, therefore anything that `git checkout` can resolve. + #[clap(long, verbatim_doc_comment)] + pub spirv_builder_version: Option, + + /// Force `rustc_codegen_spirv` to be rebuilt. + #[clap(long)] + pub rebuild_codegen: bool, + + /// Assume "yes" to "Install Rust toolchain: [y/n]" prompt. + #[clap(long, action)] + pub auto_install_rust_toolchain: bool, + + /// Clear target dir of `rustc_codegen_spirv` build after a successful build, saves about + /// 200MiB of disk space. + #[clap(long = "no-clear-target", default_value = "true", action = clap::ArgAction::SetFalse)] + pub clear_target: bool, + + /// There is a tricky situation where a shader crate that depends on workspace config can have + /// a different `Cargo.lock` lockfile version from the the workspace's `Cargo.lock`. This can + /// prevent builds when an old Rust toolchain doesn't recognise the newer lockfile version. + /// + /// The ideal way to resolve this would be to match the shader crate's toolchain with the + /// workspace's toolchain. However, that is not always possible. Another solution is to + /// `exclude = [...]` the problematic shader crate from the workspace. This also may not be a + /// suitable solution if there are a number of shader crates all sharing similar config and + /// you don't want to have to copy/paste and maintain that config across all the shaders. + /// + /// So a somewhat hacky workaround is to have `cargo gpu` overwrite lockfile versions. Enabling + /// this flag will only come into effect if there are a mix of v3/v4 lockfiles. It will also + /// only overwrite versions for the duration of a build. It will attempt to return the versions + /// to their original values once the build is finished. However, of course, unexpected errors + /// can occur and the overwritten values can remain. Hence why this behaviour is not enabled by + /// default. + /// + /// This hack is possible because the change from v3 to v4 only involves a minor change to the + /// way source URLs are encoded. See these PRs for more details: + /// * + /// * + #[clap(long, action, verbatim_doc_comment)] + pub force_overwrite_lockfiles_v4_to_v3: bool, +} + +/// Represents a functional backend installation, whether it was cached or just installed. +#[derive(Clone, Debug)] +pub struct InstalledBackend { + /// path to the `rustc_codegen_spirv` dylib + pub rustc_codegen_spirv_location: PathBuf, + /// toolchain channel name + pub toolchain_channel: String, + /// directory with target-specs json files + pub target_spec_dir: PathBuf, +} + +impl InstalledBackend { + /// Configures the supplied [`SpirvBuilder`]. `SpirvBuilder.target` must be set and must not change after calling this function. + pub fn configure_spirv_builder(&self, builder: &mut SpirvBuilder) -> anyhow::Result<()> { + builder.rustc_codegen_spirv_location = Some(self.rustc_codegen_spirv_location.clone()); + builder.toolchain_overwrite = Some(self.toolchain_channel.clone()); + builder.path_to_target_spec = Some(self.target_spec_dir.join(format!( + "{}.json", + builder.target.as_ref().context("expect target to be set")? + ))); + Ok(()) + } +} + +impl Default for Install { + #[inline] + fn default() -> Self { + Self { + shader_crate: PathBuf::from("./"), + spirv_builder_source: None, + spirv_builder_version: None, + rebuild_codegen: false, + auto_install_rust_toolchain: false, + clear_target: true, + force_overwrite_lockfiles_v4_to_v3: false, + } + } } impl Install { /// Create the `rustc_codegen_spirv_dummy` crate that depends on `rustc_codegen_spirv` fn write_source_files(source: &SpirvSource, checkout: &Path) -> anyhow::Result<()> { // skip writing a dummy project if we use a local rust-gpu checkout - if matches!(source, SpirvSource::Path { .. }) { + if source.is_path() { return Ok(()); } log::debug!( @@ -47,10 +144,12 @@ impl Install { } SpirvSource::Git { url, rev } => format!("git = \"{url}\"\nrev = \"{rev}\""), SpirvSource::Path { - rust_gpu_repo_root: rust_gpu_path, + rust_gpu_repo_root, version, } => { - let mut new_path = rust_gpu_path.to_owned(); + // this branch is currently unreachable, as we just build `rustc_codegen_spirv` directly, + // since we don't need the `dummy` crate to make cargo download it for us + let mut new_path = rust_gpu_repo_root.to_owned(); new_path.push("crates/spirv-builder"); format!("path = \"{new_path}\"\nversion = \"{version}\"") } @@ -73,25 +172,75 @@ package = "rustc_codegen_spirv" Ok(()) } - /// Add the target spec files to the crate. - fn write_target_spec_files(&self) -> anyhow::Result<()> { - for (filename, contents) in TARGET_SPECS { - let path = target_spec_dir() - .context("creating target spec dir")? - .join(filename); - if !path.is_file() || self.spirv_install.rebuild_codegen { - let mut file = std::fs::File::create(&path) - .with_context(|| format!("creating file at [{}]", path.display()))?; - file.write_all(contents.as_bytes()) - .context("writing to file")?; + /// Copy spec files from one dir to another, assuming no subdirectories + fn copy_spec_files(src: &Path, dst: &Path) -> anyhow::Result<()> { + info!( + "Copy target specs from {:?} to {:?}", + src.display(), + dst.display() + ); + std::fs::create_dir_all(dst)?; + let dir = std::fs::read_dir(src)?; + for dir_entry in dir { + let file = dir_entry?; + let file_path = file.path(); + if file_path.is_file() { + std::fs::copy(file_path, dst.join(file.file_name()))?; } } Ok(()) } + /// Add the target spec files to the crate. + fn update_spec_files( + &self, + source: &SpirvSource, + install_dir: &Path, + dummy_metadata: &Metadata, + skip_rebuild: bool, + ) -> anyhow::Result { + let mut target_specs_dst = install_dir.join("target-specs"); + if !skip_rebuild { + if let Ok(target_specs) = + dummy_metadata.find_package("rustc_codegen_spirv-target-specs") + { + let target_specs_src = target_specs + .manifest_path + .as_std_path() + .parent() + .and_then(|root| { + let src = root.join("target-specs"); + src.is_dir().then_some(src) + }) + .context("Could not find `target-specs` directory within `rustc_codegen_spirv-target-specs` dependency")?; + if source.is_path() { + // skip copy + target_specs_dst = target_specs_src; + } else { + // copy over the target-specs + Self::copy_spec_files(&target_specs_src, &target_specs_dst) + .context("copying target-specs json files")?; + } + } else { + // use legacy target specs bundled with cargo gpu + if source.is_path() { + // This is a stupid situation: + // * We can't be certain that there are `target-specs` in the local checkout (there may be some in `spirv-builder`) + // * We can't dump our legacy ones into the `install_dir`, as that would modify the local rust-gpu checkout + // -> do what the old cargo gpu did, one global dir for all target specs + // and hope parallel runs don't shred each other + target_specs_dst = cache_dir()?.join("legacy-target-specs-for-local-checkout"); + } + write_legacy_target_specs(&target_specs_dst, self.rebuild_codegen)?; + } + } + + Ok(target_specs_dst) + } + /// Install the binary pair and return the `(dylib_path, toolchain_channel)`. #[expect(clippy::too_many_lines, reason = "it's fine")] - pub fn run(&mut self) -> anyhow::Result<(PathBuf, String)> { + pub fn run(&self) -> anyhow::Result { // Ensure the cache dir exists let cache_dir = cache_dir()?; log::info!("cache directory is '{}'", cache_dir.display()); @@ -100,12 +249,11 @@ package = "rustc_codegen_spirv" })?; let source = SpirvSource::new( - &self.spirv_install.shader_crate, - self.spirv_install.spirv_builder_source.as_deref(), - self.spirv_install.spirv_builder_version.as_deref(), + &self.shader_crate, + self.spirv_builder_source.as_deref(), + self.spirv_builder_version.as_deref(), )?; - let source_is_path = matches!(source, SpirvSource::Path { .. }); - let checkout = source.install_dir()?; + let install_dir = source.install_dir()?; let dylib_filename = format!( "{}rustc_codegen_spirv{}", @@ -114,60 +262,70 @@ package = "rustc_codegen_spirv" ); let dest_dylib_path; - if source_is_path { - dest_dylib_path = checkout + if source.is_path() { + dest_dylib_path = install_dir .join("target") .join("release") .join(&dylib_filename); } else { - dest_dylib_path = checkout.join(&dylib_filename); + dest_dylib_path = install_dir.join(&dylib_filename); if dest_dylib_path.is_file() { log::info!( "cargo-gpu artifacts are already installed in '{}'", - checkout.display() + install_dir.display() ); } } - let skip_rebuild = - !source_is_path && dest_dylib_path.is_file() && !self.spirv_install.rebuild_codegen; + // if `source` is a path, always rebuild + let skip_rebuild = !source.is_path() && dest_dylib_path.is_file() && !self.rebuild_codegen; if skip_rebuild { log::info!("...and so we are aborting the install step."); } else { - Self::write_source_files(&source, &checkout).context("writing source files")?; + Self::write_source_files(&source, &install_dir).context("writing source files")?; } // TODO cache toolchain channel in a file? log::debug!("resolving toolchain version to use"); - let rustc_codegen_spirv = get_package_from_crate(&checkout, "rustc_codegen_spirv") - .context("get `rustc_codegen_spirv` metadata")?; + let dummy_metadata = query_metadata(&install_dir) + .context("resolving toolchain version: get `rustc_codegen_spirv_dummy` metadata")?; + let rustc_codegen_spirv = dummy_metadata.find_package("rustc_codegen_spirv").context( + "resolving toolchain version: expected a dependency on `rustc_codegen_spirv`", + )?; let toolchain_channel = - get_channel_from_rustc_codegen_spirv_build_script(&rustc_codegen_spirv) - .context("read toolchain from `rustc_codegen_spirv`'s build.rs")?; + get_channel_from_rustc_codegen_spirv_build_script(rustc_codegen_spirv).context( + "resolving toolchain version: read toolchain from `rustc_codegen_spirv`'s build.rs", + )?; log::info!("selected toolchain channel `{toolchain_channel:?}`"); + log::debug!("update_spec_files"); + let target_spec_dir = self + .update_spec_files(&source, &install_dir, &dummy_metadata, skip_rebuild) + .context("writing target spec files")?; + if !skip_rebuild { log::debug!("ensure_toolchain_and_components_exist"); crate::install_toolchain::ensure_toolchain_and_components_exist( &toolchain_channel, - self.spirv_install.auto_install_rust_toolchain, + self.auto_install_rust_toolchain, ) .context("ensuring toolchain and components exist")?; // to prevent unsupported version errors when using older toolchains - if !source_is_path { + if !source.is_path() { log::debug!("remove Cargo.lock"); - std::fs::remove_file(checkout.join("Cargo.lock")).context("remove Cargo.lock")?; + std::fs::remove_file(install_dir.join("Cargo.lock")) + .context("remove Cargo.lock")?; } crate::user_output!("Compiling `rustc_codegen_spirv` from source {}\n", source,); let mut build_command = std::process::Command::new("cargo"); build_command - .current_dir(&checkout) + .current_dir(&install_dir) .arg(format!("+{toolchain_channel}")) .args(["build", "--release"]) .env_remove("RUSTC"); - if source_is_path { + if source.is_path() { build_command.args(["-p", "rustc_codegen_spirv", "--lib"]); } @@ -187,15 +345,15 @@ package = "rustc_codegen_spirv" }) .context("running build command")?; - let target = checkout.join("target"); + let target = install_dir.join("target"); let dylib_path = target.join("release").join(&dylib_filename); if dylib_path.is_file() { log::info!("successfully built {}", dylib_path.display()); - if !source_is_path { + if !source.is_path() { std::fs::rename(&dylib_path, &dest_dylib_path) .context("renaming dylib path")?; - if self.spirv_install.clear_target { + if self.clear_target { log::warn!("clearing target dir {}", target.display()); std::fs::remove_dir_all(&target).context("clearing target dir")?; } @@ -204,13 +362,12 @@ package = "rustc_codegen_spirv" log::error!("could not find {}", dylib_path.display()); anyhow::bail!("`rustc_codegen_spirv` build failed"); } - - log::debug!("write_target_spec_files"); - self.write_target_spec_files() - .context("writing target spec files")?; } - self.spirv_install.dylib_path.clone_from(&dest_dylib_path); - Ok((dest_dylib_path, toolchain_channel)) + Ok(InstalledBackend { + rustc_codegen_spirv_location: dest_dylib_path, + toolchain_channel, + target_spec_dir, + }) } } diff --git a/crates/cargo-gpu/src/legacy_target_specs.rs b/crates/cargo-gpu/src/legacy_target_specs.rs new file mode 100644 index 0000000..cabcdec --- /dev/null +++ b/crates/cargo-gpu/src/legacy_target_specs.rs @@ -0,0 +1,26 @@ +//! Legacy target specs are spec jsons for versions before `rustc_codegen_spirv-target-specs` +//! came bundled with them. Instead, cargo gpu needs to bundle these legacy spec files. Luckily, +//! they are the same for all versions, as bundling target specs with the codegen backend was +//! introduced before the first target spec update. + +use anyhow::Context as _; +use log::info; +use std::path::Path; + +/// Extract legacy target specs from our executable into some directory +pub fn write_legacy_target_specs(target_spec_dir: &Path, rebuild: bool) -> anyhow::Result<()> { + info!( + "Writing legacy target specs to {}", + target_spec_dir.display() + ); + std::fs::create_dir_all(target_spec_dir)?; + for (filename, contents) in legacy_target_specs::TARGET_SPECS { + let path = target_spec_dir.join(filename); + if !path.is_file() || rebuild { + std::fs::write(&path, contents.as_bytes()).with_context(|| { + format!("writing legacy target spec file at [{}]", path.display()) + })?; + } + } + Ok(()) +} diff --git a/crates/cargo-gpu/src/main.rs b/crates/cargo-gpu/src/main.rs index 852766f..9aae856 100644 --- a/crates/cargo-gpu/src/main.rs +++ b/crates/cargo-gpu/src/main.rs @@ -55,11 +55,11 @@ use clap::Parser as _; use install::Install; use show::Show; -mod args; mod build; mod config; mod install; mod install_toolchain; +mod legacy_target_specs; mod linkage; mod lockfile; mod metadata; @@ -128,8 +128,8 @@ fn run() -> anyhow::Result<()> { match cli.command { Command::Install(install) => { - let shader_crate_path = install.spirv_install.shader_crate; - let mut command = + let shader_crate_path = install.shader_crate; + let command = config::Config::clap_command_with_cargo_config(&shader_crate_path, env_args)?; log::debug!( "installing with final merged arguments: {:#?}", @@ -138,16 +138,16 @@ fn run() -> anyhow::Result<()> { command.install.run()?; } Command::Build(build) => { - let shader_crate_path = build.install.spirv_install.shader_crate; + let shader_crate_path = build.install.shader_crate; let mut command = config::Config::clap_command_with_cargo_config(&shader_crate_path, env_args)?; log::debug!("building with final merged arguments: {command:#?}"); - if command.build_args.watch { + if command.build.watch { // When watching, do one normal run to setup the `manifest.json` file. - command.build_args.watch = false; + command.build.watch = false; command.run()?; - command.build_args.watch = true; + command.build.watch = true; command.run()?; } else { command.run()?; @@ -202,13 +202,6 @@ fn cache_dir() -> anyhow::Result { }) } -/// Location of the target spec metadata files -fn target_spec_dir() -> anyhow::Result { - let dir = cache_dir()?.join("target-specs"); - std::fs::create_dir_all(&dir)?; - Ok(dir) -} - /// Convenience function for internal use. Dumps all the CLI usage instructions. Useful for /// updating the README. fn dump_full_usage_for_readme() -> anyhow::Result<()> { diff --git a/crates/cargo-gpu/src/spirv_source.rs b/crates/cargo-gpu/src/spirv_source.rs index 32fde8d..28e93bd 100644 --- a/crates/cargo-gpu/src/spirv_source.rs +++ b/crates/cargo-gpu/src/spirv_source.rs @@ -7,7 +7,7 @@ use anyhow::Context as _; use cargo_metadata::camino::{Utf8Path, Utf8PathBuf}; use cargo_metadata::semver::Version; -use cargo_metadata::{MetadataCommand, Package}; +use cargo_metadata::{Metadata, MetadataCommand, Package}; use std::fs; use std::path::{Path, PathBuf}; @@ -96,8 +96,9 @@ impl SpirvSource { /// Look into the shader crate to get the version of `rust-gpu` it's using. pub fn get_rust_gpu_deps_from_shader(shader_crate_path: &Path) -> anyhow::Result { - let spirv_std_package = get_package_from_crate(shader_crate_path, "spirv-std")?; - let spirv_source = Self::parse_spirv_std_source_and_version(&spirv_std_package)?; + let crate_metadata = query_metadata(shader_crate_path)?; + let spirv_std_package = crate_metadata.find_package("spirv-std")?; + let spirv_source = Self::parse_spirv_std_source_and_version(spirv_std_package)?; log::debug!( "Parsed `SpirvSource` from crate `{}`: \ {spirv_source:?}", @@ -121,6 +122,11 @@ impl SpirvSource { } } + /// Returns true if self is a Path + pub const fn is_path(&self) -> bool { + matches!(self, Self::Path { .. }) + } + /// Parse a string like: /// `spirv-std v0.9.0 (https://github.com/Rust-GPU/rust-gpu?rev=54f6978c#54f6978c) (*)` /// Which would return: @@ -175,46 +181,41 @@ impl SpirvSource { } } -/// Make sure shader crate path is absolute and canonical. -fn crate_path_canonical(shader_crate_path: &Path) -> anyhow::Result { - let mut canonical_path = shader_crate_path.to_path_buf(); - - if !canonical_path.is_absolute() { - let cwd = std::env::current_dir().context("no cwd")?; - canonical_path = cwd.join(canonical_path); - } - canonical_path = canonical_path - .canonicalize() - .context("could not get absolute path to shader crate")?; - - if !canonical_path.is_dir() { - log::error!("{shader_crate_path:?} is not a directory, aborting"); - anyhow::bail!("{shader_crate_path:?} is not a directory"); - } - Ok(canonical_path) -} - /// get the Package metadata from some crate -pub fn get_package_from_crate(crate_path: &Path, crate_name: &str) -> anyhow::Result { - let canonical_crate_path = crate_path_canonical(crate_path)?; - - log::debug!( - "Running `cargo metadata` on `{}` to query for package `{crate_name}`", - canonical_crate_path.display() - ); +pub fn query_metadata(crate_path: &Path) -> anyhow::Result { + log::debug!("Running `cargo metadata` on `{}`", crate_path.display()); let metadata = MetadataCommand::new() - .current_dir(&canonical_crate_path) + .current_dir( + &crate_path + .canonicalize() + .context("could not get absolute path to shader crate")?, + ) .exec()?; + Ok(metadata) +} + +/// implements [`Self::find_package`] +pub trait FindPackage { + /// Search for a package or return a nice error + fn find_package(&self, crate_name: &str) -> anyhow::Result<&Package>; +} - let Some(package) = metadata - .packages - .into_iter() - .find(|package| package.name.eq(crate_name)) - else { - anyhow::bail!("`{crate_name}` not found in `Cargo.toml` at `{canonical_crate_path:?}`"); - }; - log::trace!(" found `{}` version `{}`", package.name, package.version); - Ok(package) +impl FindPackage for Metadata { + fn find_package(&self, crate_name: &str) -> anyhow::Result<&Package> { + if let Some(package) = self + .packages + .iter() + .find(|package| package.name.eq(crate_name)) + { + log::trace!(" found `{}` version `{}`", package.name, package.version); + Ok(package) + } else { + anyhow::bail!( + "`{crate_name}` not found in `Cargo.toml` at `{:?}`", + self.workspace_root + ); + } + } } /// Parse the `rust-toolchain.toml` in the working tree of the checked-out version of the `rust-gpu` repo. @@ -252,7 +253,7 @@ mod test { source, SpirvSource::Git { url: "https://github.com/Rust-GPU/rust-gpu".to_owned(), - rev: "82a0f69008414f51d59184763146caa6850ac588".to_owned() + rev: "86fc48032c4cd4afb74f1d81ae859711d20386a1".to_owned() } ); } @@ -274,6 +275,6 @@ mod test { .to_str() .map(std::string::ToString::to_string) .unwrap(); - assert_eq!("https___github_com_Rust-GPU_rust-gpu+82a0f690", &name); + assert_eq!("https___github_com_Rust-GPU_rust-gpu+86fc4803", &name); } } diff --git a/crates/shader-crate-template/Cargo.lock b/crates/shader-crate-template/Cargo.lock index 8a7a320..89add75 100644 --- a/crates/shader-crate-template/Cargo.lock +++ b/crates/shader-crate-template/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "autocfg" @@ -68,10 +68,11 @@ dependencies = [ [[package]] name = "spirv-std" version = "0.9.0" -source = "git+https://github.com/Rust-GPU/rust-gpu?rev=82a0f69#82a0f69008414f51d59184763146caa6850ac588" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=86fc48032c4cd4afb74f1d81ae859711d20386a1#86fc48032c4cd4afb74f1d81ae859711d20386a1" dependencies = [ "bitflags", "glam", + "libm", "num-traits", "spirv-std-macros", "spirv-std-types", @@ -80,7 +81,7 @@ dependencies = [ [[package]] name = "spirv-std-macros" version = "0.9.0" -source = "git+https://github.com/Rust-GPU/rust-gpu?rev=82a0f69#82a0f69008414f51d59184763146caa6850ac588" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=86fc48032c4cd4afb74f1d81ae859711d20386a1#86fc48032c4cd4afb74f1d81ae859711d20386a1" dependencies = [ "proc-macro2", "quote", @@ -91,7 +92,7 @@ dependencies = [ [[package]] name = "spirv-std-types" version = "0.9.0" -source = "git+https://github.com/Rust-GPU/rust-gpu?rev=82a0f69#82a0f69008414f51d59184763146caa6850ac588" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=86fc48032c4cd4afb74f1d81ae859711d20386a1#86fc48032c4cd4afb74f1d81ae859711d20386a1" [[package]] name = "syn" diff --git a/crates/shader-crate-template/Cargo.toml b/crates/shader-crate-template/Cargo.toml index faae1f2..9ed5a5a 100644 --- a/crates/shader-crate-template/Cargo.toml +++ b/crates/shader-crate-template/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["rlib", "cdylib"] # Dependencies for CPU and GPU code [dependencies] # TODO: use a simple crate version once v0.10.0 is released -spirv-std = { git = "https://github.com/Rust-GPU/rust-gpu", rev = "82a0f69" } +spirv-std = { git = "https://github.com/Rust-GPU/rust-gpu", rev = "86fc48032c4cd4afb74f1d81ae859711d20386a1" } # Dependencies for GPU code [target.'cfg(target_arch = "spirv")'.dependencies]