diff --git a/crate2nix/src/config.rs b/crate2nix/src/config.rs index 25eb8c55..114572bb 100644 --- a/crate2nix/src/config.rs +++ b/crate2nix/src/config.rs @@ -10,6 +10,8 @@ use std::{ path::Path, }; +use crate::CommitHash; + impl Config { /// Read config from path. pub fn read_from_or_default(path: &Path) -> Result { @@ -112,7 +114,7 @@ pub enum Source { /// E.g. https://github.com/kolloch/crate2nix.git url: url::Url, /// The revision hash. - rev: String, + rev: CommitHash, /// The sha256 of the fetched result. sha256: String, }, diff --git a/crate2nix/src/lib.rs b/crate2nix/src/lib.rs index 72812c6f..d06e4578 100644 --- a/crate2nix/src/lib.rs +++ b/crate2nix/src/lib.rs @@ -42,6 +42,8 @@ pub mod sources; pub mod test; pub mod util; +pub use util::CommitHash; + /// The resolved build info and the input for rendering the build.nix.tera template. #[derive(Debug, Deserialize, Serialize)] pub struct BuildInfo { diff --git a/crate2nix/src/main.rs b/crate2nix/src/main.rs index d1dbc7a5..ed201002 100644 --- a/crate2nix/src/main.rs +++ b/crate2nix/src/main.rs @@ -1,3 +1,4 @@ +use crate2nix::CommitHash; use std::path::{Path, PathBuf}; use structopt::clap::ArgGroup; use structopt::StructOpt; @@ -273,8 +274,8 @@ pub enum SourceAddingCommands { /// E.g. https://github.com/kolloch/crate2nix.git url: url::Url, - #[structopt(long = "rev", parse(from_str), help = "The git revision hash.")] - rev: String, + #[structopt(long = "rev", parse(try_from_str = CommitHash::try_from), help = "The git revision hash.")] + rev: CommitHash, }, #[structopt( diff --git a/crate2nix/src/prefetch.rs b/crate2nix/src/prefetch.rs index 7357e1b5..a83a6caa 100644 --- a/crate2nix/src/prefetch.rs +++ b/crate2nix/src/prefetch.rs @@ -314,7 +314,8 @@ impl PrefetchableSource for RegistrySource { impl PrefetchableSource for GitSource { fn needs_prefetch(&self) -> bool { - self.sha256.is_none() + // self.rev is sufficient for reproducible fetching, and that field is mandatory + false } fn prefetch(&self) -> Result { @@ -333,7 +334,7 @@ impl PrefetchableSource for GitSource { self.url.as_str(), "--fetch-submodules", "--rev", - &self.rev, + self.rev.as_ref(), ]; // TODO: --branch-name isn't documented in nix-prefetch-git --help diff --git a/crate2nix/src/resolve.rs b/crate2nix/src/resolve.rs index 4153c83e..649d4436 100644 --- a/crate2nix/src/resolve.rs +++ b/crate2nix/src/resolve.rs @@ -20,6 +20,7 @@ use std::path::{Path, PathBuf}; use crate::metadata::IndexedMetadata; #[cfg(test)] use crate::test; +use crate::CommitHash; use crate::GenerateConfig; use itertools::Itertools; use std::{collections::btree_map::BTreeMap, fmt::Display}; @@ -421,7 +422,7 @@ pub struct RegistrySource { #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Hash)] pub struct GitSource { pub url: Url, - pub rev: String, + pub rev: CommitHash, pub r#ref: Option, pub sha256: Option, } @@ -493,14 +494,43 @@ impl ResolvedSource { ); } let mut url = url::Url::parse(&source_string[GIT_SOURCE_PREFIX.len()..])?; - let mut query_pairs = url.query_pairs(); - let branch = query_pairs - .find(|(k, _)| k == "branch") - .map(|(_, v)| v.to_string()); - let rev = if let Some((_, rev)) = query_pairs.find(|(k, _)| k == "rev") { - rev.to_string() - } else if let Some(rev) = url.fragment() { - rev.to_string() + let query_pairs = url.query_pairs().collect::>(); + + // Locked git sources have optional ?branch, ?tag, ?rev, or ?ref query arguments. It is + // important to capture these in case the given commit hash is not reachable from the + // repo's default HEAD. OTOH if no form of ref is given that is an implication by cargo + // that the default HEAD should be fetched. + const REF_PARAMS: [(&str, &str); 4] = [ + ("branch", "refs/heads/"), + ("tag", "refs/tags/"), + ("rev", ""), + ("ref", ""), + ]; + let r#ref = REF_PARAMS.iter().find_map(|(key, ref_prefix)| { + let v = query_pairs.get(*key)?; + if CommitHash::parse(v).is_some() { + // Rev is usually a commit hash, but in some cases it can be a ref. Use as a ref + // only if it is **not** a valid commit hash. + return None; + } + Some(format!("{ref_prefix}{v}")) + }); + + // In locked sources the git commit hash is given as a URL fragment. It sometimes also + // given in a ?rev query argument. But in other cases a ?rev argument might not be a commit + // hash: the [cargo reference docs][] give an example of a rev argument of `"refs/pull/493/head"`. + // + // [cargo reference docs]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html + // + // That example applies to a Cargo.toml manifest - but such rev arguments do seem to be + // preserved in the lock file. + let rev = if let Some(rev) = url.fragment().and_then(CommitHash::parse) { + rev + } else if let Some(rev) = query_pairs + .get("rev") + .and_then(|rev| CommitHash::parse(rev)) + { + rev } else { return ResolvedSource::fallback_to_local_directory( config, @@ -509,12 +539,13 @@ impl ResolvedSource { "No git revision found.", ); }; + url.set_query(None); url.set_fragment(None); Ok(ResolvedSource::Git(GitSource { url, rev, - r#ref: branch, + r#ref, sha256: None, })) } diff --git a/crate2nix/src/sources.rs b/crate2nix/src/sources.rs index 4ce96358..a9eae712 100644 --- a/crate2nix/src/sources.rs +++ b/crate2nix/src/sources.rs @@ -4,6 +4,7 @@ use crate::{ config, prefetch::PrefetchableSource, resolve::{CratesIoSource, GitSource, RegistrySource}, + CommitHash, }; use anyhow::{bail, format_err, Context, Error}; use semver::Version; @@ -59,7 +60,7 @@ pub fn registry_source( } /// Returns the completed Source::Git definition by prefetching the hash. -pub fn git_io_source(url: Url, rev: String) -> Result { +pub fn git_io_source(url: Url, rev: CommitHash) -> Result { let prefetchable = GitSource { url: url.clone(), rev: rev.clone(), diff --git a/crate2nix/src/util.rs b/crate2nix/src/util.rs index 9eed2a70..593247d4 100644 --- a/crate2nix/src/util.rs +++ b/crate2nix/src/util.rs @@ -1,7 +1,11 @@ //! Homeless code. Usually abstract and algorithmic. +use core::{convert::AsRef, fmt::Display}; use std::collections::BTreeSet; +use anyhow::anyhow; +use serde::{Deserialize, Serialize}; + /// Return all occurrences of each item after the first. /// ``` /// use crate2nix::util::find_duplicates; @@ -14,3 +18,49 @@ pub fn find_duplicates<'a, T: Ord>(source: impl Iterator) -> Vec<& let mut seen = BTreeSet::new(); source.filter(|v| !seen.insert(*v)).collect() } + +/// Newtype for a string that has been verified to be a git commit hash, and has been normalized. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Deserialize, Serialize)] +#[serde(try_from = "String")] +pub struct CommitHash(String); + +impl CommitHash { + /// If the string contains 40 hexadecimal characters returns a normalized string by trimming + /// leading and trailing whitespace, and converting alphabetical characters to lower case. + pub fn parse(input: &str) -> Option { + let normalized = input.trim().to_lowercase(); + if normalized.len() == 40 && normalized.chars().all(|c| c.is_ascii_hexdigit()) { + Some(CommitHash(normalized)) + } else { + None + } + } +} + +impl AsRef for CommitHash { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl TryFrom for CommitHash { + type Error = anyhow::Error; + + fn try_from(value: String) -> Result { + >::try_from(&value) + } +} + +impl TryFrom<&str> for CommitHash { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result { + CommitHash::parse(value).ok_or_else(|| anyhow!("value {value} is not a git commit hash")) + } +} + +impl Display for CommitHash { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", self.0) + } +} diff --git a/crate2nix/templates/Cargo.nix.tera b/crate2nix/templates/Cargo.nix.tera index 8637ef07..92d04f60 100644 --- a/crate2nix/templates/Cargo.nix.tera +++ b/crate2nix/templates/Cargo.nix.tera @@ -156,12 +156,13 @@ rec { src = lib.cleanSourceWith { filter = sourceFilter; src = {{crate.source.LocalDirectory.path | safe}}; }; {%- elif crate.source.Git %} workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = {{crate.source.Git.url}}; rev = {{crate.source.Git.rev}}; - {%- if crate.source.Git.sha256 %} - sha256 = {{ crate.source.Git.sha256 }}; + {%- if crate.source.Git.ref %} + ref = {{ crate.source.Git.ref }}; {%- endif %} + submodules = true; }; {%- else %} src = builtins.throw ''ERROR: Could not resolve source: {{crate.source | json_encode() | safe}}''; diff --git a/sample_projects/bin_with_git_branch_dep/crate-hashes.json b/sample_projects/bin_with_git_branch_dep/crate-hashes.json deleted file mode 100644 index 68abb2b6..00000000 --- a/sample_projects/bin_with_git_branch_dep/crate-hashes.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "nix-base32 0.1.2-alpha.0 (git+https://github.com/kolloch/nix-base32?branch=branch-for-test#42f5544e51187f0c7535d453fcffb4b524c99eb2)": "011f945b48xkilkqbvbsxazspz5z23ka0s90ms4jiqjbhiwll1nw" -} \ No newline at end of file diff --git a/sample_projects/bin_with_git_submodule_dep/Cargo.nix b/sample_projects/bin_with_git_submodule_dep/Cargo.nix index fad04a12..ee8265ec 100644 --- a/sample_projects/bin_with_git_submodule_dep/Cargo.nix +++ b/sample_projects/bin_with_git_submodule_dep/Cargo.nix @@ -485,10 +485,10 @@ rec { edition = "2018"; links = "rocksdb"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/rust-rocksdb/rust-rocksdb"; rev = "66f04df013b6e6bd42b5a8c353406e09a7c7da2a"; - sha256 = "1rchvjrjamdaznx26gy4bmjj10rrf00mgc1wvkc489r9z1nh4h1h"; + submodules = true; }; authors = [ "Karl Hobley " @@ -837,10 +837,10 @@ rec { version = "0.21.0"; edition = "2018"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/rust-rocksdb/rust-rocksdb"; rev = "66f04df013b6e6bd42b5a8c353406e09a7c7da2a"; - sha256 = "1rchvjrjamdaznx26gy4bmjj10rrf00mgc1wvkc489r9z1nh4h1h"; + submodules = true; }; authors = [ "Tyler Neely " diff --git a/sample_projects/bin_with_git_submodule_dep/crate-hashes.json b/sample_projects/bin_with_git_submodule_dep/crate-hashes.json deleted file mode 100644 index b080ff73..00000000 --- a/sample_projects/bin_with_git_submodule_dep/crate-hashes.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "librocksdb-sys 0.15.0+8.9.1 (git+https://github.com/rust-rocksdb/rust-rocksdb#66f04df013b6e6bd42b5a8c353406e09a7c7da2a)": "1rchvjrjamdaznx26gy4bmjj10rrf00mgc1wvkc489r9z1nh4h1h", - "rocksdb 0.21.0 (git+https://github.com/rust-rocksdb/rust-rocksdb#66f04df013b6e6bd42b5a8c353406e09a7c7da2a)": "1rchvjrjamdaznx26gy4bmjj10rrf00mgc1wvkc489r9z1nh4h1h" -} \ No newline at end of file diff --git a/sample_projects/bin_with_lib_git_dep/crate-hashes.json b/sample_projects/bin_with_lib_git_dep/crate-hashes.json deleted file mode 100644 index 8691c7e5..00000000 --- a/sample_projects/bin_with_lib_git_dep/crate-hashes.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "nix-base32 0.1.2-alpha.0 (git+https://github.com/kolloch/nix-base32?rev=42f5544e51187f0c7535d453fcffb4b524c99eb2#42f5544e51187f0c7535d453fcffb4b524c99eb2)": "011f945b48xkilkqbvbsxazspz5z23ka0s90ms4jiqjbhiwll1nw" -} \ No newline at end of file diff --git a/sample_projects/codegen/Cargo.nix b/sample_projects/codegen/Cargo.nix index b25e3f07..e9657c2d 100644 --- a/sample_projects/codegen/Cargo.nix +++ b/sample_projects/codegen/Cargo.nix @@ -249,10 +249,10 @@ rec { version = "0.9.7"; edition = "2018"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/diwic/dbus-rs.git"; rev = "618262f5e3217cdd173d46d705bbac26c5141e21"; - sha256 = "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs"; + submodules = true; }; authors = [ "David Henningsson " @@ -287,10 +287,10 @@ rec { edition = "2018"; crateBin = []; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/diwic/dbus-rs.git"; rev = "618262f5e3217cdd173d46d705bbac26c5141e21"; - sha256 = "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs"; + submodules = true; }; authors = [ "David Henningsson " @@ -362,10 +362,10 @@ rec { edition = "2015"; links = "dbus"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/diwic/dbus-rs.git"; rev = "618262f5e3217cdd173d46d705bbac26c5141e21"; - sha256 = "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs"; + submodules = true; }; authors = [ "David Henningsson " diff --git a/sample_projects/codegen/crate-hashes.json b/sample_projects/codegen/crate-hashes.json deleted file mode 100644 index b395d569..00000000 --- a/sample_projects/codegen/crate-hashes.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dbus 0.9.7 (git+https://github.com/diwic/dbus-rs.git#618262f5e3217cdd173d46d705bbac26c5141e21)": "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs", - "dbus-codegen 0.10.0 (git+https://github.com/diwic/dbus-rs.git#618262f5e3217cdd173d46d705bbac26c5141e21)": "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs", - "libdbus-sys 0.2.5 (git+https://github.com/diwic/dbus-rs.git#618262f5e3217cdd173d46d705bbac26c5141e21)": "0gvhz2knd1k799l7ssh4rdm5qw0vhazzr3bxpmlgq7fhy6hjazrs" -} \ No newline at end of file diff --git a/sample_projects/sub_dir_crates/Cargo.nix b/sample_projects/sub_dir_crates/Cargo.nix index 52aaa247..5ab94883 100644 --- a/sample_projects/sub_dir_crates/Cargo.nix +++ b/sample_projects/sub_dir_crates/Cargo.nix @@ -93,10 +93,10 @@ rec { version = "0.1.0"; edition = "2018"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/kolloch/with_sub_crates.git"; rev = "f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973"; - sha256 = "0nlw7rg28p6bya040cbipq4jdcdp4h3q9shdjygfk2xkva9bjl8w"; + submodules = true; }; authors = [ "Peter Kolloch " @@ -108,10 +108,10 @@ rec { version = "0.1.0"; edition = "2018"; workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit { url = "https://github.com/kolloch/with_sub_crates.git"; rev = "f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973"; - sha256 = "0nlw7rg28p6bya040cbipq4jdcdp4h3q9shdjygfk2xkva9bjl8w"; + submodules = true; }; authors = [ "Peter Kolloch " diff --git a/sample_projects/sub_dir_crates/crate-hashes.json b/sample_projects/sub_dir_crates/crate-hashes.json deleted file mode 100644 index e81908ae..00000000 --- a/sample_projects/sub_dir_crates/crate-hashes.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "lib1 0.1.0 (git+https://github.com/kolloch/with_sub_crates.git?rev=f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973#f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973)": "0nlw7rg28p6bya040cbipq4jdcdp4h3q9shdjygfk2xkva9bjl8w", - "lib2 0.1.0 (git+https://github.com/kolloch/with_sub_crates.git?rev=f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973#f8ad2b98ff0eb5fea4962f55e3ced5b0b5afe973)": "0nlw7rg28p6bya040cbipq4jdcdp4h3q9shdjygfk2xkva9bjl8w" -} \ No newline at end of file diff --git a/tools.nix b/tools.nix index 3a867d4d..bc135c1a 100644 --- a/tools.nix +++ b/tools.nix @@ -187,24 +187,53 @@ rec { l = lib.splitString "=" s; key = builtins.elemAt l 0; in - { - # Cargo supports using the now-obsoleted "ref" key in place of - # "branch"; see cargo-vendor source - name = - if key == "ref" - then "branch" - else key; - value = builtins.elemAt l 1; - }; + { name = key; value = builtins.elemAt l 1; }; queryParams = builtins.listToAttrs (map kv queryParamsList); + firstNonNull = lib.lists.findFirst (v: v != null) null; + ref = + let + refParams = [ + { key = "branch"; refPrefix = "refs/heads/"; } + { key = "tag"; refPrefix = "refs/tags/"; } + { key = "rev"; refPrefix = ""; } + { key = "ref"; refPrefix = ""; } + ]; + parseRef = { key, refPrefix }: + let + v = if queryParams ? key then queryParams.key else null; + isActuallyAHash = parseCommitHash v == null; + in + # Rev is usually a commit hash, but in some cases it can be a ref. + # Use as a ref only if it is **not** a valid commit hash. + if v == null || isActuallyAHash then null else "${refPrefix}${v}"; + in + firstNonNull + (builtins.map parseRef refParams); + rev = + let + fromFragment = parseCommitHash fragment; + fromRev = if queryParams ? rev then parseCommitHash queryParams.rev else null; + in + firstNonNull [ fromFragment fromRev ]; in assert builtins.length splitHash <= 2; assert builtins.length splitQuestion <= 2; + assert rev != null; queryParams // { + inherit ref rev; url = preQueryParams; - urlFragment = fragment; }; + # If the input is a valid git commit hash returns a normalized version by + # converting alphabetical characters to lower case. If the input is not a + # valid hash returns null. + parseCommitHash = str: + let + normalized = lib.toLower str; + isValidHash = !(isNull (builtins.match "^[0123456789abcdef]{40}$" normalized)); + in + if builtins.isString str && isValidHash then normalized else null; + gatherLockFiles = crateDir: let fromCrateDir = @@ -263,16 +292,11 @@ rec { let parsed = parseGitSource source; src = builtins.fetchGit ({ + inherit (parsed) url rev; submodules = true; - inherit (parsed) url; - rev = - if isNull parsed.urlFragment - then parsed.rev - else parsed.urlFragment; - } // (if (parsed ? branch || parsed ? tag) - then { ref = parsed.branch or "refs/tags/${parsed.tag}"; } - else { allRefs = true; }) - ); + } // lib.optionalAttrs (!(isNull parsed.ref)) { + inherit (parsed) ref; + }); hash = pkgs.runCommand "hash-of-${attrs.name}" { nativeBuildInputs = [ pkgs.nix ]; } '' echo -n "$(nix-hash --type sha256 --base32 ${src})" > $out ''; @@ -390,18 +414,16 @@ rec { "git" = { name, version, source, ... } @ package: assert (sourceType package) == "git"; let - packageId = toPackageId package; - sha256 = extendedHashes.${packageId}; parsed = parseGitSource source; - src = pkgs.fetchgit { - name = "${name}-${version}"; - inherit sha256; - inherit (parsed) url; - rev = - if isNull parsed.urlFragment - then parsed.rev - else parsed.urlFragment; + srcname = "${name}-${version}"; + src-spec = { + inherit (parsed) url rev; + name = srcname; + submodules = true; + } // lib.optionalAttrs (!(isNull parsed.ref)) { + inherit (parsed) ref; }; + src = builtins.fetchGit src-spec; rootCargo = builtins.fromTOML (builtins.readFile "${src}/Cargo.toml"); isWorkspace = rootCargo ? "workspace"; @@ -425,7 +447,7 @@ rec { else "."; in - pkgs.runCommand (lib.removeSuffix ".tar.gz" src.name) { } + pkgs.runCommand (lib.removeSuffix ".tar.gz" srcname) { } '' mkdir -p $out cp -apR ${src}/${pathToExtract}/* $out