Adding support for private registries

Private registries were currently handled by generating code that was referencing the cache in ~/.cargo. This commit adds support to generate correct code for them.

Author: P-E-Meunier

crate2nix/default.nix

@@ -13,6 +13,7 @@
 , defaultCrateOverrides ? pkgs.defaultCrateOverrides
 , nix ? pkgs.nix
 , cargo ? pkgs.cargo
+, libsecret ? pkgs.libsecret
 , callPackage ? pkgs.callPackage
 , nix-prefetch-git ? pkgs.nix-prefetch-git
   # Seperate arguements that are NOT filled by callPackage.
@@ -26,7 +27,7 @@ let
       baseName = builtins.baseNameOf (builtins.toString name);
     in
       !(baseName == "templates" && type == "directory");
-  crate2nix = cargoNix.rootCrate.build.override {
+  crate2nix = (cargoNix.rootCrate.build.override {
     testCrateFlags = [
       "--skip nix_integration_tests"
     ];
@@ -46,7 +47,11 @@ let
         buildInputs = lib.optionals stdenv.isDarwin [ darwin.apple_sdk.frameworks.Security ];
       };
     };
-  };
+  }).overrideAttrs (attrs: {
+    postInstall = lib.optionalString stdenv.isLinux ''
+      patchelf --add-needed ${libsecret}/lib/libsecret-1.so.0 $out/bin/crate2nix
+    '';
+  });
   set_templates = if release then "" else "--set TEMPLATES_DIR ${./templates}";
 in
 symlinkJoin {

crate2nix/src/config.rs

@@ -94,6 +94,17 @@ pub enum Source {
         /// The sha256 hash of the source.
         sha256: String,
     },
+    /// Get the source from crates.io.
+    Registry {
+        /// The registry's URL
+        registry: String,
+        /// The crate name.
+        name: String,
+        /// The exact crate version to fetch.
+        version: semver::Version,
+        /// The sha256 hash of the source.
+        sha256: String,
+    },
     /// Get the source from git.
     Git {
         /// The URL of the git repository.
@@ -179,6 +190,20 @@ impl Display for Source {
                 version,
                 sha256,
             } => write!(f, "{} {} from crates.io: {}", name, version, sha256),
+            Source::Registry {
+                name,
+                version,
+                sha256,
+                registry,
+                ..
+            } => write!(
+                f,
+                "{} {} from {}: {}",
+                name,
+                version,
+                registry.to_string(),
+                sha256
+            ),
             Source::Git { url, rev, sha256 } => write!(f, "{}#{} via git: {}", url, rev, sha256),
             Source::Nix { file, attr: None } => write!(f, "{}", file),
             Source::Nix {
@@ -198,6 +223,15 @@ impl Source {
                 version,
                 ..
             } => format!("cratesIo --name '{}' '{}' '{}'", name, crate_name, version),
+            Source::Registry {
+                name: crate_name,
+                version,
+                registry,
+                ..
+            } => format!(
+                "registry --registry '{}' --name '{}' '{}' '{}'",
+                registry, name, crate_name, version
+            ),
             Source::Git { url, rev, .. } => {
                 format!("git --name '{}' '{}' --rev {}", name, url, rev)
             }

crate2nix/src/lib.rs

@@ -49,6 +49,8 @@ pub struct BuildInfo {
     pub root_package_id: Option<PackageId>,
     /// Workspaces member package IDs by package names.
     pub workspace_members: BTreeMap<String, PackageId>,
+    /// Registries used by the crates.
+    pub registries: BTreeMap<String, String>,
     /// Build info for all crates needed for this build.
     pub crates: Vec<CrateDerivation>,
     /// For convenience include the source for tests.
@@ -87,6 +89,8 @@ impl BuildInfo {
 
         prefetch_and_fill_crates_sha256(config, &merged, &mut default_nix)?;
 
+        prefetch_and_fill_registries(config, &mut default_nix)?;
+
         Ok(default_nix)
     }
 
@@ -145,6 +149,7 @@ impl BuildInfo {
                         .map(|pkg| (pkg.name.clone(), pkg_id.clone()))
                 })
                 .collect(),
+            registries: BTreeMap::new(),
             crates: metadata
                 .pkgs_by_id
                 .values()
@@ -215,6 +220,14 @@ fn prefetch_and_fill_crates_sha256(
     Ok(())
 }
 
+/// Prefetch hashes when necessary.
+fn prefetch_and_fill_registries(config: &GenerateConfig, default_nix: &mut BuildInfo) -> Result<(), Error> {
+    default_nix.registries = prefetch::prefetch_registries(config, &mut default_nix.crates)
+        .map_err(|e| format_err!("while prefetching crates for calculating sha256: {}", e))?;
+
+    Ok(())
+}
+
 fn extract_hashes_from_lockfile(
     config: &GenerateConfig,
     merged: &MergedMetadata,
@@ -299,6 +312,9 @@ pub struct GenerateConfig {
     /// The path of the `crate-hashes.json` file which is used to look up hashes and/or store
     /// prefetched hashes at.
     pub crate_hashes_json: PathBuf,
+    /// The path of the `registry-hashes.json` file which is used to look up hashes and/or store
+    /// prefetched hashes at.
+    pub registry_hashes_json: PathBuf,
     /// The nix expression for the nixpkgs path to use.
     pub nixpkgs_path: String,
     /// Additional arguments to pass to `cargo metadata`.

crate2nix/src/main.rs

@@ -98,6 +98,15 @@ pub enum Opt {
         )]
         crate_hashes: Option<PathBuf>,
 
+        #[structopt(
+            short = "r",
+            long = "registry-hashes",
+            parse(from_os_str),
+            help = "The path to the registry hash cache file. \
+                    Uses 'registry-hashes.json' in the same directory as the Cargo.nix output by default."
+        )]
+        registry_hashes: Option<PathBuf>,
+
         // Mostly useful for testing
         #[structopt(
             long = "no-cargo-lock-checksums",
@@ -364,6 +373,7 @@ fn main() -> anyhow::Result<()> {
             output: opt_output,
             nixpkgs_path,
             crate_hashes,
+            registry_hashes,
             all_features,
             default_features,
             no_default_features,
@@ -397,6 +407,13 @@ fn main() -> anyhow::Result<()> {
                     .join("crate-hashes.json")
             });
 
+            let registry_hashes_json = registry_hashes.unwrap_or_else(|| {
+                output
+                    .parent()
+                    .expect("Cargo.nix has parent")
+                    .join("registry-hashes.json")
+            });
+
             let generate_info = crate2nix::GenerateInfo::default();
 
             let feature_metadata_options = || {
@@ -446,6 +463,7 @@ fn main() -> anyhow::Result<()> {
                 output: output.clone(),
                 nixpkgs_path,
                 crate_hashes_json,
+                registry_hashes_json,
                 other_metadata_options: feature_metadata_options()?,
                 use_cargo_lock_checksums: !no_cargo_lock_checksums,
                 read_crate_hashes: !dont_read_crate_hashes,

crate2nix/src/prefetch.rs

@@ -4,7 +4,7 @@ use std::io::Write;
 use std::process::Command;
 
 use crate::metadata::PackageIdShortener;
-use crate::resolve::{CrateDerivation, CratesIoSource, GitSource, ResolvedSource};
+use crate::resolve::{CrateDerivation, CratesIoSource, GitSource, RegistrySource, ResolvedSource};
 use crate::GenerateConfig;
 use anyhow::bail;
 use anyhow::format_err;
@@ -172,6 +172,65 @@ pub fn prefetch(
     Ok(hashes)
 }
 
+/// Prefetch the config.json file from all the derivation's private registries.
+pub fn prefetch_registries(
+    config: &GenerateConfig,
+    crate_derivations: &[CrateDerivation],
+) -> Result<BTreeMap<String, String>, Error> {
+
+    let hashes_string: String = if config.read_crate_hashes {
+        std::fs::read_to_string(&config.registry_hashes_json).unwrap_or_else(|_| "{}".to_string())
+    } else {
+        "{}".to_string()
+    };
+
+    let old_prefetched_hashes: BTreeMap<String, String> = serde_json::from_str(&hashes_string)?;
+
+    let mut hashes = old_prefetched_hashes.clone();
+
+    for package in crate_derivations {
+        let registry =
+            if let ResolvedSource::Registry(RegistrySource { ref registry, .. }) = package.source {
+                registry
+            } else {
+                continue;
+            };
+        use std::collections::btree_map::Entry;
+        if let Entry::Vacant(e) = hashes.entry(registry.to_string()) {
+            eprintln!("Prefetching {} config", e.key());
+            let out = get_command_output(
+                "nix-prefetch-url",
+                &[&format!(
+                    "{}{}config.json",
+                    e.key(),
+                    if e.key().ends_with("/") { "" } else { "/" }
+                )],
+            )?;
+            e.insert(out);
+        }
+    }
+
+    if hashes != old_prefetched_hashes {
+        std::fs::write(
+            &config.registry_hashes_json,
+            serde_json::to_vec_pretty(&hashes)?,
+        )
+        .map_err(|e| {
+            format_err!(
+                "while writing hashes to {}: {}",
+                config.crate_hashes_json.to_str().unwrap_or("<unknown>"),
+                e
+            )
+        })?;
+        eprintln!(
+            "Wrote hashes to {}.",
+            config.registry_hashes_json.to_string_lossy()
+        );
+    }
+
+    Ok(hashes)
+}
+
 fn get_command_output(cmd: &str, args: &[&str]) -> Result<String, Error> {
     let output = Command::new(cmd)
         .args(args)
@@ -206,6 +265,7 @@ impl ResolvedSource {
     fn inner_prefetchable(&self) -> Option<&dyn PrefetchableSource> {
         match self {
             ResolvedSource::CratesIo(source) => Some(source),
+            ResolvedSource::Registry(source) => Some(source),
             ResolvedSource::Git(source) => Some(source),
             _ => None,
         }
@@ -241,6 +301,18 @@ impl PrefetchableSource for CratesIoSource {
     }
 }
 
+impl PrefetchableSource for RegistrySource {
+    fn needs_prefetch(&self) -> bool {
+        self.sha256.is_none()
+    }
+
+    fn prefetch(&self) -> Result<String, Error> {
+	// This is done in two steps, currently only implemented in
+	// the generated Nix.
+        unimplemented!()
+    }
+}
+
 impl PrefetchableSource for GitSource {
     fn needs_prefetch(&self) -> bool {
         self.sha256.is_none()