rustPlatform.importCargoLock: init

This function can be used to create an output path that is a cargo
vendor directory. In contrast to e.g. fetchCargoTarball all the
dependent crates are fetched using fixed-output derivations. The
hashes for the fixed-output derivations are gathered from the
Cargo.lock file.

Usage is very simple, e.g.:

importCargoLock {
  lockFile = ./Cargo.lock;
}

would use the lockfile from the current directory.

The implementation of this function is based on Eelco Dolstra's
import-cargo:

https://github.com/edolstra/import-cargo/blob/master/flake.nix

Compared to upstream:

- We use fetchgit in place of builtins.fetchGit.
- Sync to current cargo vendoring.

doc/languages-frameworks/rust.section.md

@@ -308,6 +308,37 @@ attributes can also be used:
   the `Cargo.lock`/`Cargo.toml` files need to be patched before
   vendoring.
 
+If a `Cargo.lock` file is available, you can alternatively use the
+`importCargoLock` function. In contrast to `fetchCargoTarball`, this
+function does not require a hash (unless git dependencies are used)
+and fetches every dependency as a separate fixed-output derivation.
+`importCargoLock` can be used as follows:
+
+```
+cargoDeps = rustPlatform.importCargoLock {
+  lockFile = ./Cargo.lock;
+};
+```
+
+If the `Cargo.lock` file includes git dependencies, then their output
+hashes need to be specified since they are not available through the
+lock file. For example:
+
+```
+cargoDeps = {
+  lockFile = ./Cargo.lock;
+  outputHashes = {
+    "rand-0.8.3" = "0ya2hia3cn31qa8894s3av2s8j5bjwb6yq92k0jsnlx7jid0jwqa";
+  };
+};
+```
+
+If you do not specify an output hash for a git dependency, building
+`cargoDeps` will fail and inform you of which crate needs to be
+added. To find the correct hash, you can first use `lib.fakeSha256` or
+`lib.fakeHash` as a stub hash. Building `cargoDeps` will then inform
+you of the correct hash.
+
 ### Hooks
 
 `rustPlatform` provides the following hooks to automate Cargo builds:

pkgs/build-support/rust/import-cargo-lock.nix

@@ -0,0 +1,167 @@
+{ fetchgit, fetchurl, lib, runCommand, cargo, jq }:
+
+{
+  # Cargo lock file
+  lockFile
+
+  # Hashes for git dependencies.
+, outputHashes ? {}
+}:
+
+let
+  # Parse a git source into different components.
+  parseGit = src:
+    let
+      parts = builtins.match ''git\+([^?]+)(\?rev=(.*))?#(.*)?'' src;
+      rev = builtins.elemAt parts 2;
+    in
+      if parts == null then null
+      else {
+        url = builtins.elemAt parts 0;
+        sha = builtins.elemAt parts 3;
+      } // lib.optionalAttrs (rev != null) { inherit rev; };
+
+  packages = (builtins.fromTOML (builtins.readFile lockFile)).package;
+
+  # There is no source attribute for the source package itself. But
+  # since we do not want to vendor the source package anyway, we can
+  # safely skip it.
+  depPackages = (builtins.filter (p: p ? "source") packages);
+
+  # Create dependent crates from packages.
+  #
+  # Force evaluation of the git SHA -> hash mapping, so that an error is
+  # thrown if there are stale hashes. We cannot rely on gitShaOutputHash
+  # being evaluated otherwise, since there could be no git dependencies.
+  depCrates = builtins.deepSeq (gitShaOutputHash) (builtins.map mkCrate depPackages);
+
+  # Map package name + version to git commit SHA for packages with a git source.
+  namesGitShas = builtins.listToAttrs (
+    builtins.map nameGitSha (builtins.filter (pkg: lib.hasPrefix "git+" pkg.source) depPackages)
+  );
+
+  nameGitSha = pkg: let gitParts = parseGit pkg.source; in {
+    name = "${pkg.name}-${pkg.version}";
+    value = gitParts.sha;
+  };
+
+  # Convert the attrset provided through the `outputHashes` argument to a
+  # a mapping from git commit SHA -> output hash.
+  #
+  # There may be multiple different packages with different names
+  # originating from the same git repository (typically a Cargo
+  # workspace). By using the git commit SHA as a universal identifier,
+  # the user does not have to specify the output hash for every package
+  # individually.
+  gitShaOutputHash = lib.mapAttrs' (nameVer: hash:
+    let
+      unusedHash = throw "A hash was specified for ${nameVer}, but there is no corresponding git dependency.";
+      rev = namesGitShas.${nameVer} or unusedHash; in {
+      name = rev;
+      value = hash;
+    }) outputHashes;
+
+  # We can't use the existing fetchCrate function, since it uses a
+  # recursive hash of the unpacked crate.
+  fetchCrate = pkg: fetchurl {
+    name = "crate-${pkg.name}-${pkg.version}.tar.gz";
+    url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
+    sha256 = pkg.checksum;
+  };
+
+  # Fetch and unpack a crate.
+  mkCrate = pkg:
+    let
+      gitParts = parseGit pkg.source;
+    in
+      if pkg.source == "registry+https://github.com/rust-lang/crates.io-index" then
+      let
+        crateTarball = fetchCrate pkg;
+      in runCommand "${pkg.name}-${pkg.version}" {} ''
+        mkdir $out
+        tar xf "${crateTarball}" -C $out --strip-components=1
+
+        # Cargo is happy with largely empty metadata.
+        printf '{"files":{},"package":"${pkg.checksum}"}' > "$out/.cargo-checksum.json"
+      ''
+      else if gitParts != null then
+      let
+        missingHash = throw ''
+          No hash was found while vendoring the git dependency ${pkg.name}-${pkg.version}. You can add
+          a hash through the `outputHashes` argument of `importCargoLock`:
+
+          outputHashes = {
+            "${pkg.name}-${pkg.version}" = "<hash>";
+          };
+
+          If you use `buildRustPackage`, you can add this attribute to the `cargoLock`
+          attribute set.
+        '';
+        sha256 = gitShaOutputHash.${gitParts.sha} or missingHash;
+        tree = fetchgit {
+          inherit sha256;
+          inherit (gitParts) url;
+          rev = gitParts.sha; # The commit SHA is always available.
+        };
+      in runCommand "${pkg.name}-${pkg.version}" {} ''
+        tree=${tree}
+        if grep --quiet '\[workspace\]' "$tree/Cargo.toml"; then
+          # If the target package is in a workspace, find the crate path
+          # using `cargo metadata`.
+          crateCargoTOML=$(${cargo}/bin/cargo metadata --format-version 1 --no-deps --manifest-path $tree/Cargo.toml | \
+            ${jq}/bin/jq -r '.packages[] | select(.name == "${pkg.name}") | .manifest_path')
+
+            if [[ ! -z $crateCargoTOML ]]; then
+              tree=$(dirname $crateCargoTOML)
+            else
+              >&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}' in the Cargo workspace in: $tree"
+              exit 1
+            fi
+        fi
+
+        cp -prvd "$tree/" $out
+        chmod u+w $out
+
+        # Cargo is happy with empty metadata.
+        printf '{"files":{},"package":null}' > "$out/.cargo-checksum.json"
+
+        # Set up configuration for the vendor directory.
+        cat > $out/.cargo-config <<EOF
+        [source."${gitParts.url}"]
+        git = "${gitParts.url}"
+        ${lib.optionalString (gitParts ? rev) "rev = \"${gitParts.rev}\""}
+        replace-with = "vendored-sources"
+        EOF
+      ''
+      else throw "Cannot handle crate source: ${pkg.source}";
+
+  vendorDir = runCommand "cargo-vendor-dir" {} ''
+    mkdir -p $out/.cargo
+
+    ln -s ${lockFile} $out/Cargo.lock
+
+    cat > $out/.cargo/config <<EOF
+    [source.crates-io]
+    replace-with = "vendored-sources"
+
+    [source.vendored-sources]
+    directory = "cargo-vendor-dir"
+    EOF
+
+    declare -A keysSeen
+
+    for crate in ${toString depCrates}; do
+      # Link the crate directory, removing the output path hash from the destination.
+      ln -s "$crate" $out/$(basename "$crate" | cut -c 34-)
+
+      if [ -e "$crate/.cargo-config" ]; then
+        key=$(sed 's/\[source\."\(.*\)"\]/\1/; t; d' < "$crate/.cargo-config")
+        if [[ -z ''${keysSeen[$key]} ]]; then
+          keysSeen[$key]=1
+          cat "$crate/.cargo-config" >> $out/.cargo/config
+        fi
+      fi
+    done
+  '';
+in
+  vendorDir

pkgs/development/compilers/rust/make-rust-platform.nix

@@ -16,6 +16,8 @@ rec {
       fetchCargoTarball rustc;
   };
 
+  importCargoLock = buildPackages.callPackage ../../../build-support/rust/import-cargo-lock.nix {};
+
   rustcSrc = callPackage ./rust-src.nix {
     inherit rustc;
   };