diff --git a/crate2nix/src/config.rs b/crate2nix/src/config.rs index dc37acda..b0bb3978 100644 --- a/crate2nix/src/config.rs +++ b/crate2nix/src/config.rs @@ -104,8 +104,10 @@ pub enum Source { url: url::Url, /// The revision hash. rev: String, + /// The reference (branch or tag). + r#ref: Option, /// The sha256 of the fetched result. - sha256: String, + sha256: Option, }, /// Get the source from a nix expression. Nix { @@ -181,7 +183,7 @@ impl Display for Source { version, sha256, } => write!(f, "{} {} from crates.io: {}", name, version, sha256), - Source::Git { url, rev, sha256 } => write!(f, "{}#{} via git: {}", url, rev, sha256), + Source::Git { url, rev, r#ref, sha256 } => write!(f, "{}#{} on {:?} via git: {:?}", url, rev, r#ref, sha256), Source::Nix { file, attr: None } => write!(f, "{}", file), Source::Nix { file, diff --git a/crate2nix/src/main.rs b/crate2nix/src/main.rs index a360beed..9059d6b0 100644 --- a/crate2nix/src/main.rs +++ b/crate2nix/src/main.rs @@ -267,6 +267,9 @@ pub enum SourceAddingCommands { #[structopt(long = "rev", parse(from_str), help = "The git revision hash.")] rev: String, + + #[structopt(long = "ref", parse(from_str), help = "The git reference (branch or tag).")] + r#ref: Option, }, #[structopt( @@ -319,8 +322,8 @@ impl SourceAddingCommands { let source = crate2nix::sources::crates_io_source(crate_name, crate_version)?; (name, source) } - SourceAddingCommands::Git { name, url, rev } => { - let source = crate2nix::sources::git_io_source(url, rev)?; + SourceAddingCommands::Git { name, url, rev, r#ref } => { + let source = crate2nix::sources::git_io_source(url, rev, r#ref)?; (name, source) } SourceAddingCommands::Nix { diff --git a/crate2nix/src/resolve.rs b/crate2nix/src/resolve.rs index e8ff2c41..858abb3f 100644 --- a/crate2nix/src/resolve.rs +++ b/crate2nix/src/resolve.rs @@ -366,11 +366,11 @@ pub enum ResolvedSource { impl From for ResolvedSource { fn from(source: crate::config::Source) -> Self { match source { - crate::config::Source::Git { url, rev, sha256 } => ResolvedSource::Git(GitSource { + crate::config::Source::Git { url, rev, r#ref, sha256 } => ResolvedSource::Git(GitSource { url, rev, - r#ref: None, - sha256: Some(sha256), + r#ref, + sha256, }), crate::config::Source::CratesIo { name, @@ -459,8 +459,8 @@ impl ResolvedSource { let mut url = url::Url::parse(&source_string[GIT_SOURCE_PREFIX.len()..])?; let mut query_pairs = url.query_pairs(); - let branch = query_pairs - .find(|(k, _)| k == "branch") + let r#ref = query_pairs + .find(|(k, _)| k == "branch" || k == "tag") .map(|(_, v)| v.to_string()); let rev = if let Some((_, rev)) = query_pairs.find(|(k, _)| k == "rev") { rev.to_string() @@ -479,7 +479,7 @@ impl ResolvedSource { Ok(ResolvedSource::Git(GitSource { url, rev, - r#ref: branch, + r#ref, sha256: None, })) } diff --git a/crate2nix/src/sources.rs b/crate2nix/src/sources.rs index 243a365d..727fc696 100644 --- a/crate2nix/src/sources.rs +++ b/crate2nix/src/sources.rs @@ -34,11 +34,11 @@ pub fn crates_io_source(name: String, version: Version) -> Result Result { +pub fn git_io_source(url: Url, rev: String, r#ref: Option) -> Result { let prefetchable = GitSource { url: url.clone(), rev: rev.clone(), - r#ref: None, + r#ref: r#ref.clone(), sha256: None, }; @@ -46,7 +46,7 @@ pub fn git_io_source(url: Url, rev: String) -> Result { let sha256 = prefetchable.prefetch()?; eprintln!("done."); - Ok(config::Source::Git { url, rev, sha256 }) + Ok(config::Source::Git { url, rev, r#ref, sha256: Some(sha256) }) } /// Operations on assmebling out-of-tree sources via nix. diff --git a/crate2nix/templates/Cargo.nix.tera b/crate2nix/templates/Cargo.nix.tera index 6a1e7c96..6c442d14 100644 --- a/crate2nix/templates/Cargo.nix.tera +++ b/crate2nix/templates/Cargo.nix.tera @@ -130,13 +130,15 @@ rec { else {{crate.source.LocalDirectory.path | safe}}; {%- elif crate.source.Git %} workspace_member = null; - src = pkgs.fetchgit { + src = builtins.fetchGit ({ url = {{crate.source.Git.url}}; rev = {{crate.source.Git.rev}}; - {%- if crate.source.Git.sha256 %} - sha256 = {{ crate.source.Git.sha256 }}; - {%- endif %} - }; + submodules = true; + } // (if isNull {% if crate.source.Git.ref %} {{crate.source.Git.ref}} {% else %} null {% endif %} then { + allRefs = true; + } else { + ref = {% if crate.source.Git.ref %} {{crate.source.Git.ref}} {% else %} null {% endif %}; + })); {%- else %} src = builtins.throw ''ERROR: Could not resolve source: {{crate.source | json_encode() | safe}}''; {%- endif -%} diff --git a/crate2nix/templates/crate2nix-sources.nix.tera b/crate2nix/templates/crate2nix-sources.nix.tera index 613d103c..33044390 100644 --- a/crate2nix/templates/crate2nix-sources.nix.tera +++ b/crate2nix/templates/crate2nix-sources.nix.tera @@ -40,11 +40,14 @@ rec { assert builtins.isString type; if type == "Git" - then pkgs.fetchgit { + then builtins.fetchGit ({ url = source.url; rev = source.rev; - sha256 = source.sha256; - } + submodules = true; + } // (if isNull source.ref + then { allRefs = true; } + else { inherit (source) ref; } + )) else if type == "CratesIo" then downloadFromCratesIo source else if type == "Nix" @@ -108,4 +111,4 @@ rec { tar -xzf ${archive} --strip-components=1 -C $out ''; }; -} \ No newline at end of file +} diff --git a/tests.nix b/tests.nix index 14f3a498..481c4462 100644 --- a/tests.nix +++ b/tests.nix @@ -759,10 +759,10 @@ in # # It is to have them directly as attributes for testing. - registryGit = pkgs.fetchgit { + registryGit = builtins.fetchGit { url = "https://github.com/rust-lang/crates.io-index"; rev = "18e3f063f594fc08a078f0de2bb3f94beed16ae2"; - sha256 = "0rpv12ifgnni55phlkb5ppmala7y3zrsc9dl8l99pbsjpqx95vmj"; + allRefs = true; }; registry = pkgs.linkFarm "crates.io-index" [ diff --git a/tools.nix b/tools.nix index 5fc8d0d6..328cad3a 100644 --- a/tools.nix +++ b/tools.nix @@ -162,20 +162,51 @@ rec { builtins.throw "unknown source type: ${source}"; # Extracts URL and rev from a git source URL. - # - # Crude, should be more robust :( parseGitSource = source: assert builtins.isString source; let - withoutGitPlus = lib.removePrefix "git+" source; - splitHash = lib.splitString "#" withoutGitPlus; - splitQuestion = lib.concatMap (lib.splitString "?") splitHash; + extractRevision = source: lib.last (lib.splitString "#" source); + extractPart = part: source: if lib.hasInfix part source then lib.last (lib.splitString part (lib.head (lib.splitString "#" source))) else null; + extractRepoUrl = source: + let + splitted = lib.head (lib.splitString "?" source); + split = lib.substring 4 (lib.stringLength splitted) splitted; + in + lib.head (lib.splitString "#" split); + + revision = extractRevision source; + rev = extractPart "?rev=" source; + tag = extractPart "?tag=" source; + branch = extractPart "?branch=" source; + url = extractRepoUrl source; in { - url = builtins.head splitQuestion; - rev = lib.last splitQuestion; + inherit revision url; + } // lib.optionalAttrs (! isNull rev) { + inherit rev; + } // lib.optionalAttrs (! isNull tag) { + inherit tag; + } // lib.optionalAttrs (! isNull branch) { + inherit branch; }; + getSrcFromGitSource = source: + let + parsed = parseGitSource source; + ref = parsed.branch or parsed.tag or null; + in builtins.fetchGit ({ + inherit (parsed) url; + rev = parsed.revision; + submodules = true; + } // (if isNull ref then { + allRefs = true; + } else { + ref = + if ref == (parsed.tag or null) + then "refs/tags/${ref}" + else ref; + })); + vendorSupport = { crateDir ? ./., ... }: rec { toPackageId = { name, version, source, ... }: @@ -231,12 +262,7 @@ rec { mkGitHash = { source, ... }@attrs: let - parsed = parseGitSource source; - src = builtins.fetchGit { - submodules = true; - inherit (parsed) url rev; - ref = attrs.branch or "master"; - }; + src = getSrcFromGitSource source; hash = pkgs.runCommand "hash-of-${attrs.name}" { nativeBuildInputs = [ pkgs.nix ]; } '' echo -n "$(nix-hash --type sha256 ${src})" > $out ''; @@ -299,19 +325,41 @@ rec { assert builtins.isString source; let parsed = parseGitSource source; + has = name: attrs: ! isNull (attrs.${name} or null); + hasTag = has "tag"; + hasRev = has "rev"; + hasBranch = has "branch"; + putBranch = (hasBranch attrs) || (hasBranch parsed); + putTag = (hasTag attrs) || (hasTag parsed); + putRev = (hasRev attrs) || (hasRev parsed); + isNewerCargo = builtins.compareVersions pkgs.cargo.version "1.53.0" > (-1); + key = if putRev then "?rev=${attrs.rev or parsed.rev}" else + if putTag then "?tag=${attrs.tag or parsed.tag}" else + if putBranch then "?branch=${attrs.branch or parsed.branch}" else + ""; in '' - [source."${parsed.url}"] + [source."${parsed.url}${key}"] git = "${parsed.url}" - rev = "${parsed.rev}" - ${lib.optionalString (isNull (builtins.match ".*\\?rev=[0-9a-z]{40}.*" source)) ''branch = "${attrs.branch or "master"}"''} + ${ + if isNewerCargo + then lib.optionalString putRev ''rev = "${attrs.rev or parsed.rev or parsed.revision}"'' + else ''rev = "${attrs.rev or parsed.rev or parsed.revision}"'' + } + ${ + if isNewerCargo + then lib.optionalString ((! putRev) && putBranch) ''branch = "${attrs.branch or parsed.branch}"'' + else lib.optionalString ((! putRev) && (! putTag)) ''branch = "${attrs.branch or parsed.branch or "master"}"'' + } + ${lib.optionalString ((! putRev) && putTag) ''tag = "${attrs.tag or parsed.tag}"''} replace-with = "vendored-sources" ''; gitSources = packagesByType."git" or [ ]; gitSourcesUnique = lib.unique gitSources; gitSourceConfigs = builtins.map gitSourceConfig gitSourcesUnique; - gitSourceConfigsString = lib.concatStrings gitSourceConfigs; + gitSourceConfigsUnique = lib.unique gitSourceConfigs; + gitSourceConfigsString = lib.concatStrings gitSourceConfigsUnique; in pkgs.writeText "vendor-config" @@ -350,21 +398,40 @@ rec { assert (sourceType package) == "git"; let packageId = toPackageId package; - sha256 = - hashes.${packageId} - or extraHashes.${packageId} - or (builtins.throw "Checksum for ${packageId} not found in crate-hashes.json"); - parsed = parseGitSource source; - src = pkgs.fetchgit { - name = "${name}-${version}"; - inherit sha256; - inherit (parsed) url rev; - }; + src = getSrcFromGitSource source; + srcName = "${name}-${version}"; + + rootCargo = builtins.fromTOML (builtins.readFile "${src}/Cargo.toml"); + isWorkspace = rootCargo ? "workspace"; + isPackage = rootCargo ? "package"; + containedCrates = rootCargo.workspace.members ++ (if isPackage then [ "." ] else [ ]); + + getCrateNameFromPath = path: + let + cargoTomlCrate = builtins.fromTOML (builtins.readFile "${src}/${path}/Cargo.toml"); + in + cargoTomlCrate.package.name; + + filteredPaths = + builtins.filter + (to_filter: getCrateNameFromPath to_filter == name) + containedCrates; + + pathToExtract = + if isWorkspace then + # Workaround for sources that have isWorkspace as true, but don't + # declare all their members in `workspace.members` + if builtins.length filteredPaths > 0 + then builtins.head filteredPaths + # This does not cover all possible cases, only is a last ditch effort + else name + else + "."; in - pkgs.runCommand (lib.removeSuffix ".tar.gz" src.name) { } + pkgs.runCommand (lib.removeSuffix ".tar.gz" srcName) { } '' mkdir -p $out - cp -apR ${src}/* $out + cp -apR ${src}/${pathToExtract}/* $out echo '{"package":null,"files":{}}' > $out/.cargo-checksum.json '';