From 964a7d453e2cb7a32fd5c8db8d1c1961d599c556 Mon Sep 17 00:00:00 2001
From: Jesse Hallett <jesse@sitr.us>
Date: Wed, 22 Jan 2025 23:41:25 -0800
Subject: [PATCH 1/3] fix flake lock references pointing to wrong nixpkgs

There a couple of nix expressions read flake.lock to load nixpkgs:

- crate2nix/default.nix
- nix/nix-test-runner.nix

The problem is that `flakeLock.nodes.nixpkgs` does not refer to the
intended nixpkgs version: it refers to the cachix input's nixpkgs input
which is quite old. The version of nixpkgs locked at the flake root is
`flakeLock.nodes.nixpkgs_6`.

Lock file node names are named somewhat
arbitrarily. If the intention is to load the version of nixpkgs that is
locked in flake.nix then it is necessary to:

1. Read `flakeLock.root` to get the name of the root node. (The name is
   "root", but it's better not to assume.)
2. Read `flakeLock.nodes.${rootNodeName}.inputs.nixpkgs` to get the name of the
   lock file node of the intended nixpkgs version.
3. Read `flakeLock.nodes.${nixpkgsNodeName}`

nix/nix-test-runner.nix also gets the wrong version of nix-test-runner
for the same reason.

This change introduces a function, `flakeInput`, that applies the steps
above.

I wanted to be able to submit a PR with working tests, but the tests
were not working so this PR also includes some test fixes. While I was
at it I fixed clippy warnings.

There is another issue that is explained in a comment that I put in
nix/nix-test-runner.nix. I have a fix for that which I will submit in
another PR.
---
 crate2nix/default.nix                         |  5 +--
 crate2nix/src/config.rs                       |  9 +----
 crate2nix/src/lib.rs                          |  2 +-
 crate2nix/src/prefetch.rs                     |  9 +----
 crate2nix/src/resolve.rs                      | 14 +++----
 crate2nix/src/test.rs                         |  6 ++-
 crate2nix/tests/self_build_up_to_date.rs      |  4 ++
 .../00_guides/70_building_fetched_sources.md  | 23 ++++++++----
 nix/devshell/flake-module.nix                 |  2 +-
 nix/flakeInput.nix                            | 30 +++++++++++++++
 nix/nix-test-runner.nix                       | 37 ++++++++++++++++---
 11 files changed, 98 insertions(+), 43 deletions(-)
 create mode 100644 nix/flakeInput.nix

diff --git a/crate2nix/default.nix b/crate2nix/default.nix
index 5a850653..78fa1dec 100644
--- a/crate2nix/default.nix
+++ b/crate2nix/default.nix
@@ -1,9 +1,6 @@
 # Provided by callPackage or also directly usable via nix-build with defaults.
 { pkgs ? (
-    let
-      flakeLock = builtins.fromJSON (builtins.readFile ../flake.lock);
-    in
-    import "${builtins.fetchTree flakeLock.nodes.nixpkgs.locked}" { }
+    import (builtins.fetchTree (import ../nix/flakeInput.nix "nixpkgs")) { }
   )
 , stdenv ? pkgs.stdenv
 , lib ? pkgs.lib
diff --git a/crate2nix/src/config.rs b/crate2nix/src/config.rs
index 4346d857..25eb8c55 100644
--- a/crate2nix/src/config.rs
+++ b/crate2nix/src/config.rs
@@ -196,14 +196,7 @@ impl Display for Source {
                 sha256,
                 registry,
                 ..
-            } => write!(
-                f,
-                "{} {} from {}: {}",
-                name,
-                version,
-                registry.to_string(),
-                sha256
-            ),
+            } => write!(f, "{} {} from {}: {}", name, version, registry, sha256),
             Source::Git { url, rev, sha256 } => write!(f, "{}#{} via git: {}", url, rev, sha256),
             Source::Nix { file, attr: None } => write!(f, "{}", file),
             Source::Nix {
diff --git a/crate2nix/src/lib.rs b/crate2nix/src/lib.rs
index 1d3a83ef..72812c6f 100644
--- a/crate2nix/src/lib.rs
+++ b/crate2nix/src/lib.rs
@@ -225,7 +225,7 @@ fn prefetch_and_fill_registries(
     config: &GenerateConfig,
     default_nix: &mut BuildInfo,
 ) -> Result<(), Error> {
-    default_nix.registries = prefetch::prefetch_registries(config, &mut default_nix.crates)
+    default_nix.registries = prefetch::prefetch_registries(config, &default_nix.crates)
         .map_err(|e| format_err!("while prefetching crates for calculating sha256: {}", e))?;
 
     Ok(())
diff --git a/crate2nix/src/prefetch.rs b/crate2nix/src/prefetch.rs
index 14ea9e32..b346b0eb 100644
--- a/crate2nix/src/prefetch.rs
+++ b/crate2nix/src/prefetch.rs
@@ -131,12 +131,7 @@ pub fn prefetch(
         let (sha256, hash_source) = if let Some(HashWithSource { sha256, source }) = hash {
             (sha256.trim().to_string(), source)
         } else {
-            eprintln!(
-                "Prefetching {:>4}/{}: {}",
-                idx,
-                without_hash_num,
-                source.to_string()
-            );
+            eprintln!("Prefetching {:>4}/{}: {}", idx, without_hash_num, source);
             idx += 1;
             (source.prefetch()?, HashSource::Prefetched)
         };
@@ -202,7 +197,7 @@ pub fn prefetch_registries(
                 &[&format!(
                     "{}{}config.json",
                     e.key(),
-                    if e.key().ends_with("/") { "" } else { "/" }
+                    if e.key().ends_with('/') { "" } else { "/" }
                 )],
             )?;
             e.insert(out);
diff --git a/crate2nix/src/resolve.rs b/crate2nix/src/resolve.rs
index 4153c83e..606b54e3 100644
--- a/crate2nix/src/resolve.rs
+++ b/crate2nix/src/resolve.rs
@@ -633,14 +633,14 @@ impl ResolvedSource {
     }
 }
 
-impl ToString for ResolvedSource {
-    fn to_string(&self) -> String {
+impl Display for ResolvedSource {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            Self::CratesIo(source) => source.to_string(),
-            Self::Registry(source) => source.to_string(),
-            Self::Git(source) => source.to_string(),
-            Self::LocalDirectory(source) => source.to_string(),
-            Self::Nix(source) => source.to_string(),
+            Self::CratesIo(source) => write!(f, "{}", source),
+            Self::Registry(source) => write!(f, "{}", source),
+            Self::Git(source) => write!(f, "{}", source),
+            Self::LocalDirectory(source) => write!(f, "{}", source),
+            Self::Nix(source) => write!(f, "{}", source),
         }
     }
 }
diff --git a/crate2nix/src/test.rs b/crate2nix/src/test.rs
index 8d1774b1..b95460fd 100644
--- a/crate2nix/src/test.rs
+++ b/crate2nix/src/test.rs
@@ -1,4 +1,7 @@
-///! Constructor functions for test data.
+//! Constructor functions for test data.
+
+#![allow(missing_docs)]
+
 use cargo_metadata::{Dependency, Metadata, Node, NodeDep, Package, PackageId, Resolve};
 use std::path::PathBuf;
 use tempdir::TempDir;
@@ -8,6 +11,7 @@ pub fn generate_config() -> crate::GenerateConfig {
     crate::GenerateConfig {
         cargo_toml: vec!["Cargo.toml".into()],
         crate_hashes_json: "crate-hashes.json".into(),
+        registry_hashes_json: "registry-hashes.json".into(),
         nixpkgs_path: "bogus-nixpkgs-path".into(),
         other_metadata_options: vec![],
         output: "Cargo.nix".into(),
diff --git a/crate2nix/tests/self_build_up_to_date.rs b/crate2nix/tests/self_build_up_to_date.rs
index 7e5d0130..5d4a2f4f 100644
--- a/crate2nix/tests/self_build_up_to_date.rs
+++ b/crate2nix/tests/self_build_up_to_date.rs
@@ -31,6 +31,7 @@ fn self_up_to_date() {
             output: PathBuf::from("./Cargo.nix"),
             nixpkgs_path: "../nix/nixpkgs.nix".to_string(),
             crate_hashes_json: PathBuf::from("./crate-hashes.json"),
+            registry_hashes_json: PathBuf::from("./registry-hashes.json"),
             other_metadata_options: vec![],
             use_cargo_lock_checksums: true,
             read_crate_hashes: true,
@@ -76,6 +77,9 @@ fn assert_up_to_date(project_dir: &Path) {
         crate_hashes_json: PathBuf::from("../")
             .join(project_dir)
             .join("./crate-hashes.json"),
+        registry_hashes_json: PathBuf::from("../")
+            .join(project_dir)
+            .join("./registry-hashes.json"),
         other_metadata_options: vec![],
         use_cargo_lock_checksums: true,
         read_crate_hashes: true,
diff --git a/docs/src/content/docs/00_guides/70_building_fetched_sources.md b/docs/src/content/docs/00_guides/70_building_fetched_sources.md
index 74a8d87d..5a175973 100644
--- a/docs/src/content/docs/00_guides/70_building_fetched_sources.md
+++ b/docs/src/content/docs/00_guides/70_building_fetched_sources.md
@@ -17,14 +17,21 @@ using the [tools.nix](./31_auto_generating) support:
 # nix/nix-test-runner.nix
 let
   # Reuses the locked flake inputs.
-  flakeLock = builtins.fromJSON (builtins.readFile ../flake.lock);
+  flakeInput = import ./flakeInput.nix;
   # Gets the locked sources.
-  src = builtins.fetchTree flakeLock.nodes.nix-test-runner.locked;
+  src = builtins.fetchTree (flakeInput "nix-test-runner");
+
+  # Use last pinned crate2nix packages and corresponding nixpkgs to build the
+  # test runner so that it works even if we have broken stuff!
+  crate2nix_stable = builtins.fetchTree (flakeInput "crate2nix_stable");
+  nixpkgs_stable = builtins.fetchTree (flakeInput "crate2nix_stable.nixpkgs");
 in
-{ pkgs ? import ./nixpkgs.nix { }
-  # Use last pinned crate2nix packages to build the test runner
-  # so that it works even if we have broken stuff!
-, tools ? pkgs.callPackage "${builtins.fetchTree flakeLock.nodes.crate2nix_stable.locked}/tools.nix" { }
+{
+  # A system must be specified if using default value for pkgs and calling this
+  # package from a pure evaluation context, such as from the flake devShell.
+  system ? builtins.currentSystem
+, pkgs ? import nixpkgs_stable { inherit system; }
+, tools ? pkgs.callPackage "${crate2nix_stable}/tools.nix" { }
 }:
 let
   nixTestRunner = tools.appliedCargoNix {
@@ -38,9 +45,9 @@ nixTestRunner.rootCrate.build
 ```nix
 # nix/nixpkgs.nix
 let
-  flakeLock = builtins.fromJSON (builtins.readFile ../flake.lock);
+  flakeInput = import ./lib.nix;
 in
-import "${builtins.fetchTree flakeLock.nodes.nixpkgs.locked}"
+import (builtins.fetchTree (flakeInput "nixpkgs"))
 ```
 
 ```nix
diff --git a/nix/devshell/flake-module.nix b/nix/devshell/flake-module.nix
index e39497f6..87a5b3e3 100644
--- a/nix/devshell/flake-module.nix
+++ b/nix/devshell/flake-module.nix
@@ -32,7 +32,7 @@
           { package = nix-prefetch-git; category = "nix"; }
           {
             name = "nix-test";
-            package = (import ../nix-test-runner.nix { inherit pkgs; });
+            package = (import ../nix-test-runner.nix { inherit (pkgs) system; });
             category = "nix";
             help = "nix test runner for unit tests.";
           }
diff --git a/nix/flakeInput.nix b/nix/flakeInput.nix
new file mode 100644
index 00000000..03c47e57
--- /dev/null
+++ b/nix/flakeInput.nix
@@ -0,0 +1,30 @@
+# Get a locked flake input value that can be given to `builtins.fetchTree` or to
+# `builtins.getFlake`. For example,
+#
+#     pkgs = import (builtins.fetchTree (flakeInput "nixpkgs")) { }
+#
+# This function can also be used to get inputs of inputs using dot-separated
+# paths. For example,
+#
+#     pkgs = import (builtins.fetchTree (flakeInput "crate2nix_stable.nixpkgs")) { }
+#
+# Gets the nixpkgs input of the crate2nix_stable input.
+
+let
+  flakeLock = builtins.fromJSON (builtins.readFile ../flake.lock);
+  flakeInputNodeOf = parentNode: inputName:
+    let
+      inputNodeName = builtins.getAttr inputName parentNode.inputs;
+    in
+    builtins.getAttr inputNodeName flakeLock.nodes;
+  rootNode = let rootName = flakeLock.root; in builtins.getAttr rootName flakeLock.nodes;
+in
+
+name:
+let
+  parts = builtins.split "[.]" name;
+  inputNames = builtins.filter builtins.isString parts;
+  flakeNode = builtins.foldl' flakeInputNodeOf rootNode inputNames;
+in
+flakeNode.locked
+
diff --git a/nix/nix-test-runner.nix b/nix/nix-test-runner.nix
index a366b620..02aeb0d5 100644
--- a/nix/nix-test-runner.nix
+++ b/nix/nix-test-runner.nix
@@ -1,11 +1,36 @@
 let
-  flakeLock = builtins.fromJSON (builtins.readFile ../flake.lock);
-  src = builtins.fetchTree flakeLock.nodes.nix-test-runner.locked;
+  flakeInput = import ./flakeInput.nix;
+  src = builtins.fetchTree (flakeInput "nix-test-runner");
+
+  # Use last pinned crate2nix packages and corresponding nixpkgs to build the
+  # test runner so that it works even if we have broken stuff!
+  crate2nix_stable = builtins.fetchTree (flakeInput "crate2nix_stable");
+
+  # For consistency we should use the locked version of nixpkgs from the
+  # crate2nix_stable release which would be,
+  #
+  #     nixpkgs_stable = builtins.fetchTree (flakeInput "crate2nix_stable.nixpkgs");
+  #
+  # But nix-test-runner fails to build with the crate2nix_stable and the pinned
+  # nixpkgs from the same release. That's because "crate2nix_stable.nixpkgs"
+  # provides cargo v1.78 while "nixpkgs" provides cargo v1.78. Between those two
+  # cargo versions `cargo metadata` changed the format it uses for package ID
+  # strings which breaks checksum lookups in legacy V1 Cargo.toml files in 
+  # crate2nix - and it happens that nix-test-runner uses a V1 Cargo.toml
+  # manifest.
+  #
+  # So for the time being it is necessary to use the current "nixpkgs" here
+  # which ended up on an older revision with an older cargo until there is
+  # a stable release of crate2nix that fixes V1 manifest parsing with the newer
+  # cargo, at which time this whole note may be removed.
+  nixpkgs_stable = builtins.fetchTree (flakeInput "nixpkgs");
 in
-{ pkgs ? import ./nixpkgs.nix { }
-  # Use last pinned crate2nix packages to build the test runner
-  # so that it works even if we have broken stuff!
-, tools ? pkgs.callPackage "${builtins.fetchTree flakeLock.nodes.crate2nix_stable.locked}/tools.nix" { }
+{
+  # A system must be specified if using default value for pkgs and calling this
+  # package from a pure evaluation context, such as from the flake devShell.
+  system ? builtins.currentSystem
+, pkgs ? import nixpkgs_stable { inherit system; }
+, tools ? pkgs.callPackage "${crate2nix_stable}/tools.nix" { }
 }:
 let
   nixTestRunner = tools.appliedCargoNix {

From 327d31e356bfc77cfeac7c511b1270f03735abec Mon Sep 17 00:00:00 2001
From: Jesse Hallett <jesse@sitr.us>
Date: Thu, 23 Jan 2025 00:12:06 -0800
Subject: [PATCH 2/3] fix nix file name in docs

---
 docs/src/content/docs/00_guides/70_building_fetched_sources.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/content/docs/00_guides/70_building_fetched_sources.md b/docs/src/content/docs/00_guides/70_building_fetched_sources.md
index 5a175973..629d737e 100644
--- a/docs/src/content/docs/00_guides/70_building_fetched_sources.md
+++ b/docs/src/content/docs/00_guides/70_building_fetched_sources.md
@@ -45,7 +45,7 @@ nixTestRunner.rootCrate.build
 ```nix
 # nix/nixpkgs.nix
 let
-  flakeInput = import ./lib.nix;
+  flakeInput = import ./flakeInput.nix;
 in
 import (builtins.fetchTree (flakeInput "nixpkgs"))
 ```

From cb38ed381ef191e06943111d740e077b4ce4f9a7 Mon Sep 17 00:00:00 2001
From: Jesse Hallett <jesse@sitr.us>
Date: Sun, 26 Jan 2025 19:11:31 -0800
Subject: [PATCH 3/3] revert test and lint fixes

---
 crate2nix/src/config.rs                  |  9 ++++++++-
 crate2nix/src/lib.rs                     |  2 +-
 crate2nix/src/prefetch.rs                |  9 +++++++--
 crate2nix/src/resolve.rs                 | 14 +++++++-------
 crate2nix/src/test.rs                    |  6 +-----
 crate2nix/tests/self_build_up_to_date.rs |  4 ----
 6 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/crate2nix/src/config.rs b/crate2nix/src/config.rs
index 25eb8c55..4346d857 100644
--- a/crate2nix/src/config.rs
+++ b/crate2nix/src/config.rs
@@ -196,7 +196,14 @@ impl Display for Source {
                 sha256,
                 registry,
                 ..
-            } => write!(f, "{} {} from {}: {}", name, version, registry, sha256),
+            } => write!(
+                f,
+                "{} {} from {}: {}",
+                name,
+                version,
+                registry.to_string(),
+                sha256
+            ),
             Source::Git { url, rev, sha256 } => write!(f, "{}#{} via git: {}", url, rev, sha256),
             Source::Nix { file, attr: None } => write!(f, "{}", file),
             Source::Nix {
diff --git a/crate2nix/src/lib.rs b/crate2nix/src/lib.rs
index 72812c6f..1d3a83ef 100644
--- a/crate2nix/src/lib.rs
+++ b/crate2nix/src/lib.rs
@@ -225,7 +225,7 @@ fn prefetch_and_fill_registries(
     config: &GenerateConfig,
     default_nix: &mut BuildInfo,
 ) -> Result<(), Error> {
-    default_nix.registries = prefetch::prefetch_registries(config, &default_nix.crates)
+    default_nix.registries = prefetch::prefetch_registries(config, &mut default_nix.crates)
         .map_err(|e| format_err!("while prefetching crates for calculating sha256: {}", e))?;
 
     Ok(())
diff --git a/crate2nix/src/prefetch.rs b/crate2nix/src/prefetch.rs
index b346b0eb..14ea9e32 100644
--- a/crate2nix/src/prefetch.rs
+++ b/crate2nix/src/prefetch.rs
@@ -131,7 +131,12 @@ pub fn prefetch(
         let (sha256, hash_source) = if let Some(HashWithSource { sha256, source }) = hash {
             (sha256.trim().to_string(), source)
         } else {
-            eprintln!("Prefetching {:>4}/{}: {}", idx, without_hash_num, source);
+            eprintln!(
+                "Prefetching {:>4}/{}: {}",
+                idx,
+                without_hash_num,
+                source.to_string()
+            );
             idx += 1;
             (source.prefetch()?, HashSource::Prefetched)
         };
@@ -197,7 +202,7 @@ pub fn prefetch_registries(
                 &[&format!(
                     "{}{}config.json",
                     e.key(),
-                    if e.key().ends_with('/') { "" } else { "/" }
+                    if e.key().ends_with("/") { "" } else { "/" }
                 )],
             )?;
             e.insert(out);
diff --git a/crate2nix/src/resolve.rs b/crate2nix/src/resolve.rs
index 606b54e3..4153c83e 100644
--- a/crate2nix/src/resolve.rs
+++ b/crate2nix/src/resolve.rs
@@ -633,14 +633,14 @@ impl ResolvedSource {
     }
 }
 
-impl Display for ResolvedSource {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl ToString for ResolvedSource {
+    fn to_string(&self) -> String {
         match self {
-            Self::CratesIo(source) => write!(f, "{}", source),
-            Self::Registry(source) => write!(f, "{}", source),
-            Self::Git(source) => write!(f, "{}", source),
-            Self::LocalDirectory(source) => write!(f, "{}", source),
-            Self::Nix(source) => write!(f, "{}", source),
+            Self::CratesIo(source) => source.to_string(),
+            Self::Registry(source) => source.to_string(),
+            Self::Git(source) => source.to_string(),
+            Self::LocalDirectory(source) => source.to_string(),
+            Self::Nix(source) => source.to_string(),
         }
     }
 }
diff --git a/crate2nix/src/test.rs b/crate2nix/src/test.rs
index b95460fd..8d1774b1 100644
--- a/crate2nix/src/test.rs
+++ b/crate2nix/src/test.rs
@@ -1,7 +1,4 @@
-//! Constructor functions for test data.
-
-#![allow(missing_docs)]
-
+///! Constructor functions for test data.
 use cargo_metadata::{Dependency, Metadata, Node, NodeDep, Package, PackageId, Resolve};
 use std::path::PathBuf;
 use tempdir::TempDir;
@@ -11,7 +8,6 @@ pub fn generate_config() -> crate::GenerateConfig {
     crate::GenerateConfig {
         cargo_toml: vec!["Cargo.toml".into()],
         crate_hashes_json: "crate-hashes.json".into(),
-        registry_hashes_json: "registry-hashes.json".into(),
         nixpkgs_path: "bogus-nixpkgs-path".into(),
         other_metadata_options: vec![],
         output: "Cargo.nix".into(),
diff --git a/crate2nix/tests/self_build_up_to_date.rs b/crate2nix/tests/self_build_up_to_date.rs
index 5d4a2f4f..7e5d0130 100644
--- a/crate2nix/tests/self_build_up_to_date.rs
+++ b/crate2nix/tests/self_build_up_to_date.rs
@@ -31,7 +31,6 @@ fn self_up_to_date() {
             output: PathBuf::from("./Cargo.nix"),
             nixpkgs_path: "../nix/nixpkgs.nix".to_string(),
             crate_hashes_json: PathBuf::from("./crate-hashes.json"),
-            registry_hashes_json: PathBuf::from("./registry-hashes.json"),
             other_metadata_options: vec![],
             use_cargo_lock_checksums: true,
             read_crate_hashes: true,
@@ -77,9 +76,6 @@ fn assert_up_to_date(project_dir: &Path) {
         crate_hashes_json: PathBuf::from("../")
             .join(project_dir)
             .join("./crate-hashes.json"),
-        registry_hashes_json: PathBuf::from("../")
-            .join(project_dir)
-            .join("./registry-hashes.json"),
         other_metadata_options: vec![],
         use_cargo_lock_checksums: true,
         read_crate_hashes: true,