Cannot build crate with PyO3 dependency

[Wulfsta]

I am trying to use crane to build the minimum rust-from-python example that PyO3 provides. My flake.nix is as follows:

{
  description = "Build a cargo project";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";

    crane = {
      url = "github:ipetkov/crane";
      inputs.nixpkgs.follows = "nixpkgs";
    };

    flake-utils.url = "github:numtide/flake-utils";
  };

  outputs = { self, nixpkgs, crane, flake-utils, ... }:
    flake-utils.lib.eachDefaultSystem (system:
      let
        pkgs = import nixpkgs {
          inherit system;
        };

        craneLib = crane.lib.${system};
        src = ./.;

        # Build *just* the cargo dependencies, so we can reuse
        # all of that work (e.g. via cachix) when running in CI
        cargoArtifacts = craneLib.buildDepsOnly {
          inherit src;
          buildInputs = with pkgs; [
            python3
          ];
        };

        # Build the actual crate itself, reusing the dependency
        # artifacts from above.
        py-test = craneLib.buildPackage {
          inherit cargoArtifacts src;
        };
      in
      {
        checks = {
          # Build the crate as part of `nix flake check` for convenience
          inherit py-test;

          # Run clippy (and deny all warnings) on the crate source,
          # again, resuing the dependency artifacts from above.
          #
          # Note that this is done as a separate derivation so that
          # we can block the CI if there are issues here, but not
          # prevent downstream consumers from building our crate by itself.
          py-test-clippy = craneLib.cargoClippy {
            inherit cargoArtifacts src;
            cargoClippyExtraArgs = "-- --deny warnings";
          };

          # Check formatting
          py-test-fmt = craneLib.cargoFmt {
            inherit src;
          };

          # Check code coverage (note: this will not upload coverage anywhere)
          py-test-coverage = craneLib.cargoTarpaulin {
            inherit cargoArtifacts src;
          };
        };

        packages.default = py-test;

        apps.default = flake-utils.lib.mkApp {
          drv = py-test;
        };

        devShell = pkgs.mkShell {
          inputsFrom = builtins.attrValues self.checks;

          # Extra inputs can be added here
          nativeBuildInputs = with pkgs; [
            cargo
            rustc
          ];
        };
      });
}

I'm seeing the following error from nix flake check:

nix log /nix/store/zwsin4kixagiagfbbhn6mzm8cv1amfbz-py_test-0.1.0.drv
warning: executing SQLite statement '

         create table if not exists BinaryCaches (
             id        integer primary key autoincrement not null,
             url       text unique not null,
             timestamp integer not null,
             storeDir  text not null,
             wantMassQuery integer not null,
             priority  integer not null
         );

         create table if not exists NARs (
             cache            integer not null,
             hashPart         text not null,
             namePart         text,
             url              text,
             compression      text,
             fileHash         text,
             fileSize         integer,
             narHash          text,
             narSize          integer,
             refs             text,
             deriver          text,
             sigs             text,
             ca               text,
             timestamp        integer not null,
             present          integer not null,
             primary key (cache, hashPart),
             foreign key (cache) references BinaryCaches(id) on delete cascade
         );

         create table if not exists Realisations (
             cache integer not null,
             outputId text not null,
             content blob, -- Json serialisation of the realisation, or null if the realisation is absent
             timestamp        integer not null,
             primary key (cache, outputId),
@nix { "action": "setPhase", "phase": "unpackPhase" }
unpacking sources
unpacking source archive /nix/store/drqs8ifw3kkxbai97ffnddnci3fg9cls-wpzzlw9mr6dy3q1zbv25rq8ybhn3byai-source
source root is wpzzlw9mr6dy3q1zbv25rq8ybhn3byai-source
@nix { "action": "setPhase", "phase": "patchPhase" }
patching sources
Executing configureCargoCommonVars
copying cargo artifacts from /nix/store/sfv5k81mca8l6acyqqsqimalrk177561-py_test-deps-0.1.0/target.tar.zst to target
@nix { "action": "setPhase", "phase": "configurePhase" }
configuring
will append /build/wpzzlw9mr6dy3q1zbv25rq8ybhn3byai-source/.cargo-home/config.toml with contents of /nix/store/dp4xq7rviwv7ih8nlv8lhy3grr1fhlyb-vend>
no configure script, doing nothing
setting CARGO_BUILD_RUSTFLAGS to "--remap-path-prefix /nix/store/dp4xq7rviwv7ih8nlv8lhy3grr1fhlyb-vendor-cargo-deps=/sources"
@nix { "action": "setPhase", "phase": "buildPhase" }
building
cargo 1.60.0
running: cargoBuildLog=$(mktemp cargoBuildLogXXXX.json)
cargo build --workspace --release --message-format json-render-diagnostics  >"$cargoBuildLog"

   Compiling pyo3-build-config v0.16.4
error: failed to run custom build command for `pyo3-build-config v0.16.4`

Caused by:
  process didn't exit successfully: `/build/wpzzlw9mr6dy3q1zbv25rq8ybhn3byai-source/target/release/build/pyo3-build-config-d9d1048ee5e4183e/build-sc>
  --- stdout
  cargo:rerun-if-env-changed=PYO3_CONFIG_FILE
  cargo:rerun-if-env-changed=PYO3_NO_PYTHON
  cargo:rerun-if-env-changed=PYO3_PYTHON
  cargo:rerun-if-env-changed=VIRTUAL_ENV
  cargo:rerun-if-env-changed=CONDA_PREFIX
  cargo:rerun-if-env-changed=PATH

  --- stderr
  error: no Python 3.x interpreter found

What I expect is happening is that PyO3 is not detecting the Python version. I have considered potentially using the PyO3 pyo3-build-config.txt, but wanted to ask here for your thoughts first.

[ipetkov]

Hi @Wulfsta! I think the problem is that the py-test derivation does not automatically inherit the buildInputs defined on the cargoArtifacts derivation. In other words, the python3 input is completely missing as far as the py-test derivation is concerned.

There are two ways to go about fixing this:

# First way is to explicitly add the build inputs to the `py-test` derivation (and any other derivations which will need it present):
let
  src = ./.;
  cargoArtifacts = craneLib.buildDepsOnly {
    buildInputs = with pkgs; [
      python3
    ];
  };

  # Build the actual crate itself, reusing the dependency
  # artifacts from above.
  py-test = craneLib.buildPackage {
    inherit cargoArtifacts src;
    buildInputs = with pkgs; [
      python3
    ];
  };
in {
# Rest of flake...
}

# The second way is to create a set of common arguments and pass
# them to all derivations (after adding any necessary additions).
let
  src = ./.;
  commonArgs = {
    inherit src;
    buildInputs = with pkgs; [
      python3
    ];
  };

  cargoArtifacts = craneLib.buildDepsOnly commonArgs;

  py-test = craneLib.buildPackage (commonArgs // {
    inherit cargoArtifacts;
  });
in {
# Rest of flake...
}

The README was recently updated to show the second method if you want to see a more complete example. Hope this helps!

[Wulfsta]

Thanks!

[Wulfsta]

As an additional note for this specific PyO3 case - it is useful to link $out/lib/lib<name>.so to $out/lib/<name>.so in the craneLib.buildPackage call as follows:

postFixup = ''
  ln -s $out/lib/lib<name>.so $out/lib/<name>.so
'';

This allows you to navigate to this directory, open a Python interpreter, and import the package without hassle.

Hopefully this helps anyone trying to do this in the future.