From 7a5f28390a7f9235b465e0e869e6c57836d1e5c0 Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Tue, 21 Jan 2025 19:25:39 -0800 Subject: [PATCH 01/12] do not prefetch git sources - use builtins.fetchGit to avoid need for hash --- crate2nix/src/prefetch.rs | 2 +- crate2nix/templates/Cargo.nix.tera | 8 +++++--- tools.nix | 29 ++++++++++++++++++----------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/crate2nix/src/prefetch.rs b/crate2nix/src/prefetch.rs index 14ea9e32..43bbfe9a 100644 --- a/crate2nix/src/prefetch.rs +++ b/crate2nix/src/prefetch.rs @@ -314,7 +314,7 @@ impl PrefetchableSource for RegistrySource { impl PrefetchableSource for GitSource { fn needs_prefetch(&self) -> bool { - self.sha256.is_none() + false } fn prefetch(&self) -> Result { diff --git a/crate2nix/templates/Cargo.nix.tera b/crate2nix/templates/Cargo.nix.tera index 980d5032..bf79a76a 100644 --- a/crate2nix/templates/Cargo.nix.tera +++ b/crate2nix/templates/Cargo.nix.tera @@ -156,11 +156,13 @@ rec { src = lib.cleanSourceWith { filter = sourceFilter; src = {{crate.source.LocalDirectory.path | safe}}; }; {%- elif crate.source.Git %} workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = {{crate.source.Git.url}}; rev = {{crate.source.Git.rev}}; - {%- if crate.source.Git.sha256 %} - sha256 = {{ crate.source.Git.sha256 }}; + {%- if crate.source.Git.ref %} + ref = {{ crate.source.Git.ref }}; + {%- else -%} + allRefs = true; {%- endif %} }; {%- else %} diff --git a/tools.nix b/tools.nix index 3a867d4d..a4167b0b 100644 --- a/tools.nix +++ b/tools.nix @@ -390,18 +390,25 @@ rec { "git" = { name, version, source, ... } @ package: assert (sourceType package) == "git"; let - packageId = toPackageId package; - sha256 = extendedHashes.${packageId}; parsed = parseGitSource source; - src = pkgs.fetchgit { - name = "${name}-${version}"; - inherit sha256; - inherit (parsed) url; - rev = - if isNull parsed.urlFragment - then parsed.rev - else parsed.urlFragment; + srcname = "${name}-${version}"; + ref = + if builtins.hasAttr "tag" parsed then "refs/tags/${parsed.tag}" + else if builtins.hasAttr "branch" parsed then parsed.branch + else if builtins.hasAttr "ref" parsed then parsed.ref + else null; + src-spec = { + inherit (parsed) url; + allRefs = isNull ref; + name = srcname; + rev = + if isNull parsed.urlFragment + then parsed.rev + else parsed.urlFragment; + } // lib.optionalAttrs (!(isNull ref)) { + inherit ref; }; + src = builtins.trace src-spec.rev (builtins.trace src-spec (builtins.fetchGit src-spec)); rootCargo = builtins.fromTOML (builtins.readFile "${src}/Cargo.toml"); isWorkspace = rootCargo ? "workspace"; @@ -425,7 +432,7 @@ rec { else "."; in - pkgs.runCommand (lib.removeSuffix ".tar.gz" src.name) { } + pkgs.runCommand (lib.removeSuffix ".tar.gz" srcname) { } '' mkdir -p $out cp -apR ${src}/${pathToExtract}/* $out From 2cb81acf34554fce24bcb35cca1d192488c66c06 Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sat, 25 Jan 2025 14:03:17 -0800 Subject: [PATCH 02/12] delete test crate hashes that are no longer used --- sample_projects/bin_with_git_branch_dep/crate-hashes.json | 3 --- sample_projects/bin_with_lib_git_dep/crate-hashes.json | 3 --- 2 files changed, 6 deletions(-) delete mode 100644 sample_projects/bin_with_git_branch_dep/crate-hashes.json delete mode 100644 sample_projects/bin_with_lib_git_dep/crate-hashes.json diff --git a/sample_projects/bin_with_git_branch_dep/crate-hashes.json b/sample_projects/bin_with_git_branch_dep/crate-hashes.json deleted file mode 100644 index 68abb2b6..00000000 --- a/sample_projects/bin_with_git_branch_dep/crate-hashes.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "nix-base32 0.1.2-alpha.0 (git+https://github.com/kolloch/nix-base32?branch=branch-for-test#42f5544e51187f0c7535d453fcffb4b524c99eb2)": "011f945b48xkilkqbvbsxazspz5z23ka0s90ms4jiqjbhiwll1nw" -} \ No newline at end of file diff --git a/sample_projects/bin_with_lib_git_dep/crate-hashes.json b/sample_projects/bin_with_lib_git_dep/crate-hashes.json deleted file mode 100644 index 8691c7e5..00000000 --- a/sample_projects/bin_with_lib_git_dep/crate-hashes.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "nix-base32 0.1.2-alpha.0 (git+https://github.com/kolloch/nix-base32?rev=42f5544e51187f0c7535d453fcffb4b524c99eb2#42f5544e51187f0c7535d453fcffb4b524c99eb2)": "011f945b48xkilkqbvbsxazspz5z23ka0s90ms4jiqjbhiwll1nw" -} \ No newline at end of file From 62c56a3e836e3eeeda88621f066612ae684e3137 Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sat, 25 Jan 2025 14:03:36 -0800 Subject: [PATCH 03/12] update git fetcher in tools --- tools.nix | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tools.nix b/tools.nix index a4167b0b..84e377e2 100644 --- a/tools.nix +++ b/tools.nix @@ -397,18 +397,20 @@ rec { else if builtins.hasAttr "branch" parsed then parsed.branch else if builtins.hasAttr "ref" parsed then parsed.ref else null; + rev = + if isNull parsed.rev + then parsed.urlFragment + else parsed.rev; src-spec = { - inherit (parsed) url; - allRefs = isNull ref; - name = srcname; - rev = - if isNull parsed.urlFragment - then parsed.rev - else parsed.urlFragment; - } // lib.optionalAttrs (!(isNull ref)) { + inherit (parsed) url; + allRefs = isNull ref; + name = srcname; + } // lib.optionalAttrs (!(isNull ref)) { inherit ref; + } // lib.optionalAttrs (!(isNull rev)) { + inherit rev; }; - src = builtins.trace src-spec.rev (builtins.trace src-spec (builtins.fetchGit src-spec)); + src = builtins.fetchGit src-spec; rootCargo = builtins.fromTOML (builtins.readFile "${src}/Cargo.toml"); isWorkspace = rootCargo ? "workspace"; From abd30b03e25a7f0079f34953102856374c44d687 Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sat, 25 Jan 2025 14:43:33 -0800 Subject: [PATCH 04/12] we need to fetch submodules --- crate2nix/src/prefetch.rs | 1 + crate2nix/templates/Cargo.nix.tera | 3 ++- tools.nix | 7 ++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crate2nix/src/prefetch.rs b/crate2nix/src/prefetch.rs index 43bbfe9a..193cc727 100644 --- a/crate2nix/src/prefetch.rs +++ b/crate2nix/src/prefetch.rs @@ -314,6 +314,7 @@ impl PrefetchableSource for RegistrySource { impl PrefetchableSource for GitSource { fn needs_prefetch(&self) -> bool { + // self.rev is sufficient for reproducible fetching, and that field is mandatory false } diff --git a/crate2nix/templates/Cargo.nix.tera b/crate2nix/templates/Cargo.nix.tera index bf79a76a..df8bcc27 100644 --- a/crate2nix/templates/Cargo.nix.tera +++ b/crate2nix/templates/Cargo.nix.tera @@ -159,11 +159,12 @@ rec { src = builtins.fetchGit { url = {{crate.source.Git.url}}; rev = {{crate.source.Git.rev}}; - {%- if crate.source.Git.ref %} + {% if crate.source.Git.ref %} ref = {{ crate.source.Git.ref }}; {%- else -%} allRefs = true; {%- endif %} + submodules = true; }; {%- else %} src = builtins.throw ''ERROR: Could not resolve source: {{crate.source | json_encode() | safe}}''; diff --git a/tools.nix b/tools.nix index 84e377e2..36a74ca3 100644 --- a/tools.nix +++ b/tools.nix @@ -398,13 +398,14 @@ rec { else if builtins.hasAttr "ref" parsed then parsed.ref else null; rev = - if isNull parsed.rev - then parsed.urlFragment - else parsed.rev; + if builtins.hasAttr "rev" parsed + then parsed.rev + else builtins.trace parsed parsed.urlFragment; src-spec = { inherit (parsed) url; allRefs = isNull ref; name = srcname; + submodules = true; } // lib.optionalAttrs (!(isNull ref)) { inherit ref; } // lib.optionalAttrs (!(isNull rev)) { From aec89f20ee4de746a77978b68f5f99e6c2cb61f4 Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sat, 25 Jan 2025 17:00:43 -0800 Subject: [PATCH 05/12] I think we don't want allRefs --- crate2nix/templates/Cargo.nix.tera | 4 +--- .../bin_with_git_submodule_dep/Cargo.nix | 10 ++++++---- .../bin_with_git_submodule_dep/crate-hashes.json | 4 ---- sample_projects/codegen/Cargo.nix | 15 +++++++++------ sample_projects/codegen/crate-hashes.json | 5 ----- sample_projects/sub_dir_crates/Cargo.nix | 10 ++++++---- sample_projects/sub_dir_crates/crate-hashes.json | 4 ---- tools.nix | 12 +++++------- 8 files changed, 27 insertions(+), 37 deletions(-) delete mode 100644 sample_projects/bin_with_git_submodule_dep/crate-hashes.json delete mode 100644 sample_projects/codegen/crate-hashes.json delete mode 100644 sample_projects/sub_dir_crates/crate-hashes.json diff --git a/crate2nix/templates/Cargo.nix.tera b/crate2nix/templates/Cargo.nix.tera index df8bcc27..57c5e4bc 100644 --- a/crate2nix/templates/Cargo.nix.tera +++ b/crate2nix/templates/Cargo.nix.tera @@ -161,9 +161,7 @@ rec { rev = {{crate.source.Git.rev}}; {% if crate.source.Git.ref %} ref = {{ crate.source.Git.ref }}; - {%- else -%} - allRefs = true; - {%- endif %} + {%- endif -%} submodules = true; }; {%- else %} diff --git a/sample_projects/bin_with_git_submodule_dep/Cargo.nix b/sample_projects/bin_with_git_submodule_dep/Cargo.nix index 34060349..38e6ce29 100644 --- a/sample_projects/bin_with_git_submodule_dep/Cargo.nix +++ b/sample_projects/bin_with_git_submodule_dep/Cargo.nix @@ -485,10 +485,11 @@ rec { edition = "2018"; links = "rocksdb"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/rust-rocksdb/rust-rocksdb"; rev = "66f04df013b6e6bd42b5a8c353406e09a7c7da2a"; - sha256 = "1rchvjrjamdaznx26gy4bmjj10rrf00mgc1wvkc489r9z1nh4h1h"; + + submodules = true; }; authors = [ "Karl Hobley " @@ -837,10 +838,11 @@ rec { version = "0.21.0"; edition = "2018"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/rust-rocksdb/rust-rocksdb"; rev = "66f04df013b6e6bd42b5a8c353406e09a7c7da2a"; - sha256 = "1rchvjrjamdaznx26gy4bmjj10rrf00mgc1wvkc489r9z1nh4h1h"; + + submodules = true; }; authors = [ "Tyler Neely " diff --git a/sample_projects/bin_with_git_submodule_dep/crate-hashes.json b/sample_projects/bin_with_git_submodule_dep/crate-hashes.json deleted file mode 100644 index b080ff73..00000000 --- a/sample_projects/bin_with_git_submodule_dep/crate-hashes.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "librocksdb-sys 0.15.0+8.9.1 (git+https://github.com/rust-rocksdb/rust-rocksdb#66f04df013b6e6bd42b5a8c353406e09a7c7da2a)": "1rchvjrjamdaznx26gy4bmjj10rrf00mgc1wvkc489r9z1nh4h1h", - "rocksdb 0.21.0 (git+https://github.com/rust-rocksdb/rust-rocksdb#66f04df013b6e6bd42b5a8c353406e09a7c7da2a)": "1rchvjrjamdaznx26gy4bmjj10rrf00mgc1wvkc489r9z1nh4h1h" -} \ No newline at end of file diff --git a/sample_projects/codegen/Cargo.nix b/sample_projects/codegen/Cargo.nix index 91bdd375..8896e609 100644 --- a/sample_projects/codegen/Cargo.nix +++ b/sample_projects/codegen/Cargo.nix @@ -249,10 +249,11 @@ rec { version = "0.9.7"; edition = "2018"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/diwic/dbus-rs.git"; rev = "618262f5e3217cdd173d46d705bbac26c5141e21"; - sha256 = "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs"; + + submodules = true; }; authors = [ "David Henningsson " @@ -287,10 +288,11 @@ rec { edition = "2018"; crateBin = []; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/diwic/dbus-rs.git"; rev = "618262f5e3217cdd173d46d705bbac26c5141e21"; - sha256 = "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs"; + + submodules = true; }; authors = [ "David Henningsson " @@ -362,10 +364,11 @@ rec { edition = "2015"; links = "dbus"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/diwic/dbus-rs.git"; rev = "618262f5e3217cdd173d46d705bbac26c5141e21"; - sha256 = "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs"; + + submodules = true; }; authors = [ "David Henningsson " diff --git a/sample_projects/codegen/crate-hashes.json b/sample_projects/codegen/crate-hashes.json deleted file mode 100644 index b395d569..00000000 --- a/sample_projects/codegen/crate-hashes.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dbus 0.9.7 (git+https://github.com/diwic/dbus-rs.git#618262f5e3217cdd173d46d705bbac26c5141e21)": "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs", - "dbus-codegen 0.10.0 (git+https://github.com/diwic/dbus-rs.git#618262f5e3217cdd173d46d705bbac26c5141e21)": "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs", - "libdbus-sys 0.2.5 (git+https://github.com/diwic/dbus-rs.git#618262f5e3217cdd173d46d705bbac26c5141e21)": "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs" -} \ No newline at end of file diff --git a/sample_projects/sub_dir_crates/Cargo.nix b/sample_projects/sub_dir_crates/Cargo.nix index 5b16ffb6..f412460b 100644 --- a/sample_projects/sub_dir_crates/Cargo.nix +++ b/sample_projects/sub_dir_crates/Cargo.nix @@ -93,10 +93,11 @@ rec { version = "0.1.0"; edition = "2018"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/kolloch/with_sub_crates.git"; rev = "f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973"; - sha256 = "0nlw7rg28p6bya040cbipq4jdcdp4h3q9shdjygfk2xkva9bjl8w"; + + submodules = true; }; authors = [ "Peter Kolloch " @@ -108,10 +109,11 @@ rec { version = "0.1.0"; edition = "2018"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/kolloch/with_sub_crates.git"; rev = "f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973"; - sha256 = "0nlw7rg28p6bya040cbipq4jdcdp4h3q9shdjygfk2xkva9bjl8w"; + + submodules = true; }; authors = [ "Peter Kolloch " diff --git a/sample_projects/sub_dir_crates/crate-hashes.json b/sample_projects/sub_dir_crates/crate-hashes.json deleted file mode 100644 index e81908ae..00000000 --- a/sample_projects/sub_dir_crates/crate-hashes.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "lib1 0.1.0 (git+https://github.com/kolloch/with_sub_crates.git?rev=f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973#f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973)": "0nlw7rg28p6bya040cbipq4jdcdp4h3q9shdjygfk2xkva9bjl8w", - "lib2 0.1.0 (git+https://github.com/kolloch/with_sub_crates.git?rev=f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973#f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973)": "0nlw7rg28p6bya040cbipq4jdcdp4h3q9shdjygfk2xkva9bjl8w" -} \ No newline at end of file diff --git a/tools.nix b/tools.nix index 36a74ca3..5bc0d66c 100644 --- a/tools.nix +++ b/tools.nix @@ -393,17 +393,15 @@ rec { parsed = parseGitSource source; srcname = "${name}-${version}"; ref = - if builtins.hasAttr "tag" parsed then "refs/tags/${parsed.tag}" - else if builtins.hasAttr "branch" parsed then parsed.branch - else if builtins.hasAttr "ref" parsed then parsed.ref + if parsed ? tag then "refs/tags/${parsed.tag}" + else if parsed ? branch then parsed.branch + else if parsed ? ref then parsed.ref else null; rev = - if builtins.hasAttr "rev" parsed - then parsed.rev - else builtins.trace parsed parsed.urlFragment; + if parsed ? rev then parsed.rev + else parsed.urlFragment; src-spec = { inherit (parsed) url; - allRefs = isNull ref; name = srcname; submodules = true; } // lib.optionalAttrs (!(isNull ref)) { From 0aec946f74e846549c3ced4903b938277771a80a Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sun, 26 Jan 2025 14:32:01 -0800 Subject: [PATCH 06/12] parse commit hashes to verify validity --- crate2nix/src/config.rs | 4 +++- crate2nix/src/lib.rs | 2 ++ crate2nix/src/main.rs | 5 ++-- crate2nix/src/prefetch.rs | 2 +- crate2nix/src/resolve.rs | 41 +++++++++++++++++++++++++++----- crate2nix/src/sources.rs | 3 ++- crate2nix/src/util.rs | 50 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 96 insertions(+), 11 deletions(-) diff --git a/crate2nix/src/config.rs b/crate2nix/src/config.rs index 4346d857..a5456d91 100644 --- a/crate2nix/src/config.rs +++ b/crate2nix/src/config.rs @@ -10,6 +10,8 @@ use std::{ path::Path, }; +use crate::CommitHash; + impl Config { /// Read config from path. pub fn read_from_or_default(path: &Path) -> Result { @@ -112,7 +114,7 @@ pub enum Source { /// E.g. https://github.com/kolloch/crate2nix.git url: url::Url, /// The revision hash. - rev: String, + rev: CommitHash, /// The sha256 of the fetched result. sha256: String, }, diff --git a/crate2nix/src/lib.rs b/crate2nix/src/lib.rs index 1d3a83ef..62d7f4ca 100644 --- a/crate2nix/src/lib.rs +++ b/crate2nix/src/lib.rs @@ -42,6 +42,8 @@ pub mod sources; pub mod test; pub mod util; +pub use util::CommitHash; + /// The resolved build info and the input for rendering the build.nix.tera template. #[derive(Debug, Deserialize, Serialize)] pub struct BuildInfo { diff --git a/crate2nix/src/main.rs b/crate2nix/src/main.rs index d1dbc7a5..ed201002 100644 --- a/crate2nix/src/main.rs +++ b/crate2nix/src/main.rs @@ -1,3 +1,4 @@ +use crate2nix::CommitHash; use std::path::{Path, PathBuf}; use structopt::clap::ArgGroup; use structopt::StructOpt; @@ -273,8 +274,8 @@ pub enum SourceAddingCommands { /// E.g. https://github.com/kolloch/crate2nix.git url: url::Url, - #[structopt(long = "rev", parse(from_str), help = "The git revision hash.")] - rev: String, + #[structopt(long = "rev", parse(try_from_str = CommitHash::try_from), help = "The git revision hash.")] + rev: CommitHash, }, #[structopt( diff --git a/crate2nix/src/prefetch.rs b/crate2nix/src/prefetch.rs index 193cc727..861b2c78 100644 --- a/crate2nix/src/prefetch.rs +++ b/crate2nix/src/prefetch.rs @@ -334,7 +334,7 @@ impl PrefetchableSource for GitSource { self.url.as_str(), "--fetch-submodules", "--rev", - &self.rev, + self.rev.as_ref(), ]; // TODO: --branch-name isn't documented in nix-prefetch-git --help diff --git a/crate2nix/src/resolve.rs b/crate2nix/src/resolve.rs index 4153c83e..90e483ca 100644 --- a/crate2nix/src/resolve.rs +++ b/crate2nix/src/resolve.rs @@ -20,6 +20,7 @@ use std::path::{Path, PathBuf}; use crate::metadata::IndexedMetadata; #[cfg(test)] use crate::test; +use crate::CommitHash; use crate::GenerateConfig; use itertools::Itertools; use std::{collections::btree_map::BTreeMap, fmt::Display}; @@ -421,7 +422,7 @@ pub struct RegistrySource { #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Hash)] pub struct GitSource { pub url: Url, - pub rev: String, + pub rev: CommitHash, pub r#ref: Option, pub sha256: Option, } @@ -494,13 +495,40 @@ impl ResolvedSource { } let mut url = url::Url::parse(&source_string[GIT_SOURCE_PREFIX.len()..])?; let mut query_pairs = url.query_pairs(); + + // Locked git sources have optional ?branch or ?tag or ?rev query arguments. It is + // important to capture these in case the given commit hash is not reachable from the + // repo's default HEAD. OTOH if no form of ref is given that is an implication by cargo + // that the default HEAD should be fetched. let branch = query_pairs .find(|(k, _)| k == "branch") .map(|(_, v)| v.to_string()); - let rev = if let Some((_, rev)) = query_pairs.find(|(k, _)| k == "rev") { - rev.to_string() - } else if let Some(rev) = url.fragment() { - rev.to_string() + let tag = query_pairs + .find(|(k, _)| k == "tag") + .map(|(_, v)| format!("refs/tags/{v}")); + let ref_via_rev = query_pairs + .find(|(k, _)| k == "rev") + // Rev is usually a commit hash, but in some cases it can be a ref. Use as a ref only + // if it is not a valid commit hash. + .filter(|(_, v)| CommitHash::parse(v).is_none()) + .map(|(_, v)| v.to_string()); + let r#ref = branch.or(tag).or(ref_via_rev); + + // In locked sources the git commit hash is given as a URL fragment. It sometimes also + // given in a ?rev query argument. But in other cases a ?rev argument might not be a commit + // hash: the [cargo reference docs][] give an example of a rev argument of `"refs/pull/493/head"`. + // + // [cargo reference docs]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html + // + // That example applies to a Cargo.toml manifest - but such rev arguments do seem to be + // preserved in the lock file. + let rev = if let Some(rev) = url.fragment().and_then(CommitHash::parse) { + rev + } else if let Some(rev) = query_pairs + .find(|(k, _)| k == "rev") + .and_then(|(_, rev)| CommitHash::parse(&rev)) + { + rev } else { return ResolvedSource::fallback_to_local_directory( config, @@ -509,12 +537,13 @@ impl ResolvedSource { "No git revision found.", ); }; + url.set_query(None); url.set_fragment(None); Ok(ResolvedSource::Git(GitSource { url, rev, - r#ref: branch, + r#ref, sha256: None, })) } diff --git a/crate2nix/src/sources.rs b/crate2nix/src/sources.rs index 4ce96358..a9eae712 100644 --- a/crate2nix/src/sources.rs +++ b/crate2nix/src/sources.rs @@ -4,6 +4,7 @@ use crate::{ config, prefetch::PrefetchableSource, resolve::{CratesIoSource, GitSource, RegistrySource}, + CommitHash, }; use anyhow::{bail, format_err, Context, Error}; use semver::Version; @@ -59,7 +60,7 @@ pub fn registry_source( } /// Returns the completed Source::Git definition by prefetching the hash. -pub fn git_io_source(url: Url, rev: String) -> Result { +pub fn git_io_source(url: Url, rev: CommitHash) -> Result { let prefetchable = GitSource { url: url.clone(), rev: rev.clone(), diff --git a/crate2nix/src/util.rs b/crate2nix/src/util.rs index 9eed2a70..593247d4 100644 --- a/crate2nix/src/util.rs +++ b/crate2nix/src/util.rs @@ -1,7 +1,11 @@ //! Homeless code. Usually abstract and algorithmic. +use core::{convert::AsRef, fmt::Display}; use std::collections::BTreeSet; +use anyhow::anyhow; +use serde::{Deserialize, Serialize}; + /// Return all occurrences of each item after the first. /// ``` /// use crate2nix::util::find_duplicates; @@ -14,3 +18,49 @@ pub fn find_duplicates<'a, T: Ord>(source: impl Iterator) -> Vec<& let mut seen = BTreeSet::new(); source.filter(|v| !seen.insert(*v)).collect() } + +/// Newtype for a string that has been verified to be a git commit hash, and has been normalized. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Deserialize, Serialize)] +#[serde(try_from = "String")] +pub struct CommitHash(String); + +impl CommitHash { + /// If the string contains 40 hexadecimal characters returns a normalized string by trimming + /// leading and trailing whitespace, and converting alphabetical characters to lower case. + pub fn parse(input: &str) -> Option { + let normalized = input.trim().to_lowercase(); + if normalized.len() == 40 && normalized.chars().all(|c| c.is_ascii_hexdigit()) { + Some(CommitHash(normalized)) + } else { + None + } + } +} + +impl AsRef for CommitHash { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl TryFrom for CommitHash { + type Error = anyhow::Error; + + fn try_from(value: String) -> Result { + >::try_from(&value) + } +} + +impl TryFrom<&str> for CommitHash { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result { + CommitHash::parse(value).ok_or_else(|| anyhow!("value {value} is not a git commit hash")) + } +} + +impl Display for CommitHash { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", self.0) + } +} From a0f275ee6c7c5f2a8aaa7629ca319fe4522114ca Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sun, 26 Jan 2025 14:36:14 -0800 Subject: [PATCH 07/12] remove blank lines from Cargo.nix files --- sample_projects/bin_with_git_submodule_dep/Cargo.nix | 2 -- sample_projects/codegen/Cargo.nix | 3 --- sample_projects/sub_dir_crates/Cargo.nix | 2 -- 3 files changed, 7 deletions(-) diff --git a/sample_projects/bin_with_git_submodule_dep/Cargo.nix b/sample_projects/bin_with_git_submodule_dep/Cargo.nix index 38e6ce29..e77f713b 100644 --- a/sample_projects/bin_with_git_submodule_dep/Cargo.nix +++ b/sample_projects/bin_with_git_submodule_dep/Cargo.nix @@ -488,7 +488,6 @@ rec { src = builtins.fetchGit { url = "https://github.com/rust-rocksdb/rust-rocksdb"; rev = "66f04df013b6e6bd42b5a8c353406e09a7c7da2a"; - submodules = true; }; authors = [ @@ -841,7 +840,6 @@ rec { src = builtins.fetchGit { url = "https://github.com/rust-rocksdb/rust-rocksdb"; rev = "66f04df013b6e6bd42b5a8c353406e09a7c7da2a"; - submodules = true; }; authors = [ diff --git a/sample_projects/codegen/Cargo.nix b/sample_projects/codegen/Cargo.nix index 8896e609..df258f2b 100644 --- a/sample_projects/codegen/Cargo.nix +++ b/sample_projects/codegen/Cargo.nix @@ -252,7 +252,6 @@ rec { src = builtins.fetchGit { url = "https://github.com/diwic/dbus-rs.git"; rev = "618262f5e3217cdd173d46d705bbac26c5141e21"; - submodules = true; }; authors = [ @@ -291,7 +290,6 @@ rec { src = builtins.fetchGit { url = "https://github.com/diwic/dbus-rs.git"; rev = "618262f5e3217cdd173d46d705bbac26c5141e21"; - submodules = true; }; authors = [ @@ -367,7 +365,6 @@ rec { src = builtins.fetchGit { url = "https://github.com/diwic/dbus-rs.git"; rev = "618262f5e3217cdd173d46d705bbac26c5141e21"; - submodules = true; }; authors = [ diff --git a/sample_projects/sub_dir_crates/Cargo.nix b/sample_projects/sub_dir_crates/Cargo.nix index f412460b..b9ac2f8f 100644 --- a/sample_projects/sub_dir_crates/Cargo.nix +++ b/sample_projects/sub_dir_crates/Cargo.nix @@ -96,7 +96,6 @@ rec { src = builtins.fetchGit { url = "https://github.com/kolloch/with_sub_crates.git"; rev = "f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973"; - submodules = true; }; authors = [ @@ -112,7 +111,6 @@ rec { src = builtins.fetchGit { url = "https://github.com/kolloch/with_sub_crates.git"; rev = "f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973"; - submodules = true; }; authors = [ From 9939a0a8af33ccf73458442e7bf40203fbd7ae09 Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sun, 26 Jan 2025 15:59:37 -0800 Subject: [PATCH 08/12] make git source parsing consistent in different places where it is done --- crate2nix/src/resolve.rs | 36 +++++++++--------- tools.nix | 82 ++++++++++++++++++++++++---------------- 2 files changed, 69 insertions(+), 49 deletions(-) diff --git a/crate2nix/src/resolve.rs b/crate2nix/src/resolve.rs index 90e483ca..649d4436 100644 --- a/crate2nix/src/resolve.rs +++ b/crate2nix/src/resolve.rs @@ -494,25 +494,27 @@ impl ResolvedSource { ); } let mut url = url::Url::parse(&source_string[GIT_SOURCE_PREFIX.len()..])?; - let mut query_pairs = url.query_pairs(); + let query_pairs = url.query_pairs().collect::>(); - // Locked git sources have optional ?branch or ?tag or ?rev query arguments. It is + // Locked git sources have optional ?branch, ?tag, ?rev, or ?ref query arguments. It is // important to capture these in case the given commit hash is not reachable from the // repo's default HEAD. OTOH if no form of ref is given that is an implication by cargo // that the default HEAD should be fetched. - let branch = query_pairs - .find(|(k, _)| k == "branch") - .map(|(_, v)| v.to_string()); - let tag = query_pairs - .find(|(k, _)| k == "tag") - .map(|(_, v)| format!("refs/tags/{v}")); - let ref_via_rev = query_pairs - .find(|(k, _)| k == "rev") - // Rev is usually a commit hash, but in some cases it can be a ref. Use as a ref only - // if it is not a valid commit hash. - .filter(|(_, v)| CommitHash::parse(v).is_none()) - .map(|(_, v)| v.to_string()); - let r#ref = branch.or(tag).or(ref_via_rev); + const REF_PARAMS: [(&str, &str); 4] = [ + ("branch", "refs/heads/"), + ("tag", "refs/tags/"), + ("rev", ""), + ("ref", ""), + ]; + let r#ref = REF_PARAMS.iter().find_map(|(key, ref_prefix)| { + let v = query_pairs.get(*key)?; + if CommitHash::parse(v).is_some() { + // Rev is usually a commit hash, but in some cases it can be a ref. Use as a ref + // only if it is **not** a valid commit hash. + return None; + } + Some(format!("{ref_prefix}{v}")) + }); // In locked sources the git commit hash is given as a URL fragment. It sometimes also // given in a ?rev query argument. But in other cases a ?rev argument might not be a commit @@ -525,8 +527,8 @@ impl ResolvedSource { let rev = if let Some(rev) = url.fragment().and_then(CommitHash::parse) { rev } else if let Some(rev) = query_pairs - .find(|(k, _)| k == "rev") - .and_then(|(_, rev)| CommitHash::parse(&rev)) + .get("rev") + .and_then(|rev| CommitHash::parse(rev)) { rev } else { diff --git a/tools.nix b/tools.nix index 5bc0d66c..f6d71183 100644 --- a/tools.nix +++ b/tools.nix @@ -187,24 +187,57 @@ rec { l = lib.splitString "=" s; key = builtins.elemAt l 0; in - { - # Cargo supports using the now-obsoleted "ref" key in place of - # "branch"; see cargo-vendor source - name = - if key == "ref" - then "branch" - else key; - value = builtins.elemAt l 1; - }; + { name = key; value = builtins.elemAt l 1; }; queryParams = builtins.listToAttrs (map kv queryParamsList); + firstNonNull = lib.lists.findFirst (v: v != null) null; + ref = + let + refParams = [ + { key = "branch"; refPrefix = "refs/heads/"; } + { key = "tag"; refPrefix = "refs/tags/"; } + { key = "rev"; refPrefix = ""; } + { key = "ref"; refPrefix = ""; } + ]; + parseRef = { key, refPrefix }: + let + v = if queryParams ? key then queryParams.key else null; + in + # Rev is usually a commit hash, but in some cases it can be a ref. + # Use as a ref only if it is **not** a valid commit hash. + if parseCommitHash v != null then null else "{refPrefix}{v}"; + in + firstNonNull + (builtins.map parseRef refParams); + rev = + let + fromFragment = parseCommitHash fragment; + fromRev = if queryParams ? rev then parseCommitHash queryParams.rev else null; + in + firstNonNull [ fromFragment fromRev ]; in assert builtins.length splitHash <= 2; assert builtins.length splitQuestion <= 2; + assert rev != null; queryParams // { + inherit ref rev; url = preQueryParams; - urlFragment = fragment; }; + # If the input is a valid git commit hash returns a normalized version by + # converting alphabetical characters to lower case. If the input is not a + # valid hash returns null. + parseCommitHash = str: + let + normalized = lib.toLower str; + isValidHash = !(isNull (builtins.match "^[0123456789abcdef]{40}$" normalized)); + in + if builtins.isString str && isValidHash then normalized else null; + + # Returns input unchanged if it is a non-empty string. Otherwise returns + # null. + parseCommitRef = str: + if builtins.isString str && builtins.match "^\s*$" str != null then str else null; + gatherLockFiles = crateDir: let fromCrateDir = @@ -263,16 +296,11 @@ rec { let parsed = parseGitSource source; src = builtins.fetchGit ({ + inherit (parsed) url rev; submodules = true; - inherit (parsed) url; - rev = - if isNull parsed.urlFragment - then parsed.rev - else parsed.urlFragment; - } // (if (parsed ? branch || parsed ? tag) - then { ref = parsed.branch or "refs/tags/${parsed.tag}"; } - else { allRefs = true; }) - ); + } // lib.optionalAttrs (!(isNull parsed.ref)) { + inherit (parsed) ref; + }); hash = pkgs.runCommand "hash-of-${attrs.name}" { nativeBuildInputs = [ pkgs.nix ]; } '' echo -n "$(nix-hash --type sha256 --base32 ${src})" > $out ''; @@ -392,22 +420,12 @@ rec { let parsed = parseGitSource source; srcname = "${name}-${version}"; - ref = - if parsed ? tag then "refs/tags/${parsed.tag}" - else if parsed ? branch then parsed.branch - else if parsed ? ref then parsed.ref - else null; - rev = - if parsed ? rev then parsed.rev - else parsed.urlFragment; src-spec = { - inherit (parsed) url; + inherit (parsed) url rev; name = srcname; submodules = true; - } // lib.optionalAttrs (!(isNull ref)) { - inherit ref; - } // lib.optionalAttrs (!(isNull rev)) { - inherit rev; + } // lib.optionalAttrs (!(isNull parsed.ref)) { + inherit (parsed) ref; }; src = builtins.fetchGit src-spec; From dacea5d9aec53d8989d058ef584e7ca277dfa8f7 Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sun, 26 Jan 2025 16:11:46 -0800 Subject: [PATCH 09/12] fiddle with whitespace in tera template --- crate2nix/templates/Cargo.nix.tera | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crate2nix/templates/Cargo.nix.tera b/crate2nix/templates/Cargo.nix.tera index 57c5e4bc..7d74ad03 100644 --- a/crate2nix/templates/Cargo.nix.tera +++ b/crate2nix/templates/Cargo.nix.tera @@ -159,9 +159,9 @@ rec { src = builtins.fetchGit { url = {{crate.source.Git.url}}; rev = {{crate.source.Git.rev}}; - {% if crate.source.Git.ref %} + {%- if crate.source.Git.ref %} ref = {{ crate.source.Git.ref }}; - {%- endif -%} + {%- endif %} submodules = true; }; {%- else %} From ca3fafca216336fca450206bbe79be7559562d77 Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sun, 26 Jan 2025 18:42:17 -0800 Subject: [PATCH 10/12] removed a function I didn't end up using --- tools.nix | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools.nix b/tools.nix index f6d71183..84481ddc 100644 --- a/tools.nix +++ b/tools.nix @@ -233,11 +233,6 @@ rec { in if builtins.isString str && isValidHash then normalized else null; - # Returns input unchanged if it is a non-empty string. Otherwise returns - # null. - parseCommitRef = str: - if builtins.isString str && builtins.match "^\s*$" str != null then str else null; - gatherLockFiles = crateDir: let fromCrateDir = From 10de5c9c5a69f602032360678670b86957dce3c5 Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sun, 26 Jan 2025 19:20:26 -0800 Subject: [PATCH 11/12] accidentally use rust interpolation syntax in a nix string --- tools.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.nix b/tools.nix index 84481ddc..43dd452f 100644 --- a/tools.nix +++ b/tools.nix @@ -204,7 +204,7 @@ rec { in # Rev is usually a commit hash, but in some cases it can be a ref. # Use as a ref only if it is **not** a valid commit hash. - if parseCommitHash v != null then null else "{refPrefix}{v}"; + if parseCommitHash v != null then null else "${refPrefix}${v}"; in firstNonNull (builtins.map parseRef refParams); From 9e68ee5bcf1acf0aeda704069030c02ccc240d10 Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Sun, 26 Jan 2025 20:23:00 -0800 Subject: [PATCH 12/12] missed a null check --- tools.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools.nix b/tools.nix index 43dd452f..bc135c1a 100644 --- a/tools.nix +++ b/tools.nix @@ -201,10 +201,11 @@ rec { parseRef = { key, refPrefix }: let v = if queryParams ? key then queryParams.key else null; + isActuallyAHash = parseCommitHash v == null; in # Rev is usually a commit hash, but in some cases it can be a ref. # Use as a ref only if it is **not** a valid commit hash. - if parseCommitHash v != null then null else "${refPrefix}${v}"; + if v == null || isActuallyAHash then null else "${refPrefix}${v}"; in firstNonNull (builtins.map parseRef refParams);