allow to pass outputHashes to crane (#266)

This makes it possible to evaluate crane in a nixos test without network
as well as allow to backup all fetched input derivations properly in a
binary cache, whereas fetchGit will fallback to downloading from a
repository, which also requires a `git` binary to be present.

Co-authored-by: Ivan Petkov <ivanppetkov@gmail.com>

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 * Added `devShell`, a thin wrapper around `pkgs.mkShell` which automatically
   provides `cargo` and `rustc`.
+* Added the ability to specify output hashes of git dependencies for fully
+  offline evaluations. The `outputHashes` attribute can now be optionally
+  specified in `vendorCargoDeps`, `vendorGitDeps`, `vendorMultipleCargoDeps`, or
+  anything else which delegates to them.
 
 ### Changed
 * **Breaking** (technically): `buildDepsOnly`, `buildPackage`, `cargoBuild`,

checks/default.nix

@@ -351,6 +351,19 @@ in
       pkgs.libiconv
     ];
   };
+  simpleGitWithHashes = myLib.buildPackage {
+    src = myLib.cleanCargoSource ./simple-git;
+    outputHashes = {
+      "git+https://github.com/BurntSushi/byteorder.git#8d9f0c0fb577e2d3aa0265a0b0c0b8af56780aec" = "sha256-rP/1WWuCH8dl/zLyarcdWHJvAoR1N9yiIhJHGk6EFFA=";
+      "git+https://github.com/dtolnay/rustversion.git?rev=2abd4d0e00db08bb91145cb88e5dcbad2f45bbcb#2abd4d0e00db08bb91145cb88e5dcbad2f45bbcb" = "sha256-deS6eoNuWPZ1V3XO9UzR07vLHZjT9arAYL0xEJCoU6E=";
+      "git+https://github.com/rust-lang/libc.git?branch=main#28ab9b9e7bd04a5c5aca3f4d78583214f63d4002" = "sha256-SXmFggY9AUbwzb98JUpxZ1I3DBBhKonZVRUJDrECSb0=";
+      "git+https://github.com/seanmonstar/num_cpus.git?tag=v1.13.1#5f1b03332000b4c4274b5bd35fac516049ff1c6b" = "sha256-mNMxS/WXjNokO9mFXQSwyuIpIp/n94EQ9Ni0Bl40es8";
+
+    };
+    buildInputs = lib.optionals isDarwin [
+      pkgs.libiconv
+    ];
+  };
   simpleGitWorkspaceInheritance = myLib.buildPackage {
     src = myLib.cleanCargoSource ./simple-git-workspace-inheritance;
   };

checks/vendorGitSubset.nix

@@ -5,6 +5,7 @@
 , linkFarmFromDrvs
 , runCommand
 , vendorGitDeps
+, outputHashes ? { }
 }:
 
 let
@@ -18,6 +19,7 @@ let
 
   vendoredGit = vendorGitDeps {
     lockPackages = lock.package;
+    inherit outputHashes;
   };
 
   checkSubset = runCommand "vendorGitSubsetAsExpected" { } ''

docs/API.md

@@ -154,6 +154,7 @@ environment variables during the build, you can bring them back via
 * `cargoTestCommand`
 * `cargoTestExtraArgs`
 * `dummySrc`
+* `outputHashes`
 
 ### `craneLib.buildPackage`
 
@@ -227,6 +228,7 @@ environment variables during the build, you can bring them back via
 * `cargoExtraArgs`
 * `cargoTestCommand`
 * `cargoTestExtraArgs`
+* `outputHashes`
 
 #### Native build dependencies and included hooks
 The following hooks are automatically added as native build inputs:
@@ -844,13 +846,16 @@ any crates it contains for vendoring.
 * `rev`: the exact revision to check out
 
 #### Optional attributes
+* `allRefs`: whether all git refs should be fetched in order to look for the
+  specified `rev`
+  - Default value: `true` if `ref` is set to `null`, `false` otherwise
 * `ref`: the ref (i.e. branch or tag) to which `rev` belongs to. For branches it
   should be `"refs/head/${branch}"` and for tags it should be
   `"refs/tags/${tag}"`
   - Default value: `null`
-* `allRefs`: whether all git refs should be fetched in order to look for the
-  specified `rev`
-  - Default value: `true` if `ref` is set to `null`, `false` otherwise
+* `sha256`: the sha256 hash of the (unpacked) download. If provided `fetchgit` will be used
+  (instead of `builtins.fetchGit`) which allows for offline evaluations.
+  - Default value: `null`
 
 ### `craneLib.findCargoFiles`
 
@@ -974,6 +979,7 @@ environment variables during the build, you can bring them back via
 * `cargoLockParsed`
 * `checkPhaseCargoCommand`
 * `installPhaseCommand`
+* `outputHashes`
 * `pnameSuffix`
 * `stdenv`
 
@@ -1234,6 +1240,11 @@ the vendored directories (i.e. this configuration can be appended to the
 At least one of the above attributes must be specified, or an error will be
 raised during evaluation.
 
+#### Optional attributes
+* `outputHashes`: a mapping of package-source to the sha256 of the (unpacked)
+  download. Useful for supporting fully offline evaluations.
+  - Default value: `[]`
+
 ### `craneLib.vendorCargoRegistries`
 
 `vendorCargoRegistries :: set -> set`
@@ -1275,6 +1286,11 @@ access.
 * `lockPackages`: a list of all `[[package]]` entries found in the project's
   `Cargo.lock` file (parsed via `builtins.fromTOML`)
 
+#### Optional attributes
+* `outputHashes`: a mapping of package-source to the sha256 of the (unpacked)
+  download. Useful for supporting fully offline evaluations.
+  - Default value: `[]`
+
 #### Output attributes
 * `config`: the configuration entires needed to point cargo to the vendored
   sources. This is intended to be appended to `$CARGO_HOME/config.toml` verbatim
@@ -1311,6 +1327,9 @@ the vendored directories (i.e. this configuration can be appended to the
 * `cargoLockParsedList`: a list of attrsets representing the parsed contents of
   different `Cargo.lock` files to be included while vendoring.
   - Default value: `[]`
+* `outputHashes`: a mapping of package-source to the sha256 of the (unpacked)
+  download. Useful for supporting fully offline evaluations.
+  - Default value: `[]`
 * `registries`: an attrset of registry names to their index URL. The default
   ("crates-io") registry need not be specified, as it will automatically be
   available, but it can be overridden if required.

lib/buildDepsOnly.nix

@@ -21,6 +21,7 @@ let
     "cargoExtraArgs"
     "cargoTestCommand"
     "cargoTestExtraArgs"
+    "outputHashes"
     "dummySrc"
   ];
 

lib/buildPackage.nix

@@ -23,6 +23,7 @@ let
     "cargoExtraArgs"
     "cargoTestCommand"
     "cargoTestExtraArgs"
+    "outputHashes"
   ];
 
   # Avoid recomputing values when passing args down

lib/downloadCargoPackageFromGit.nix

@@ -2,21 +2,32 @@
 , craneUtils
 , jq
 , lib
+, fetchgit
 , runCommand
 }:
 
 { git
 , rev
 , ref ? null
+, sha256 ? null
 , allRefs ? ref == null
 }:
 let
   maybeRef = lib.optionalAttrs (ref != null) { inherit ref; };
-  repo = builtins.fetchGit (maybeRef // {
-    inherit allRefs rev;
-    url = git;
-    submodules = true;
-  });
+  repo =
+    if sha256 == null then
+      builtins.fetchGit
+        (maybeRef // {
+          inherit allRefs rev;
+          url = git;
+          submodules = true;
+        })
+    else
+      fetchgit {
+        inherit rev sha256;
+        url = git;
+        fetchSubmodules = true;
+      };
 
   deps = {
     nativeBuildInputs = [

lib/mkCargoDerivation.nix

@@ -37,6 +37,7 @@ let
     "checkPhaseCargoCommand"
     "installPhaseCommand"
     "pnameSuffix"
+    "outputHashes"
     "stdenv"
   ];
 in

lib/vendorCargoDeps.nix

@@ -39,4 +39,5 @@ in
 vendorMultipleCargoDeps ({
   inherit cargoConfigs;
   cargoLockParsedList = [ lock ];
+  outputHashes = args.outputHashes or { };
 } // optionalAttrs (args ? registries) { inherit (args) registries; })

lib/vendorGitDeps.nix

@@ -4,6 +4,7 @@
 }:
 
 { lockPackages
+, outputHashes ? { }
 }:
 let
   inherit (builtins)
@@ -91,6 +92,11 @@ let
           inherit (p) git;
           inherit ref;
           rev = p.lockedRev;
+          sha256 = outputHashes.${p.package.source} or (lib.warnIf
+            (outputHashes != { })
+            "No output hash provided for ${p.package.source}"
+            null
+          );
         };
 
         # NB: we filter out any crates NOT in the lock file

lib/vendorMultipleCargoDeps.nix

@@ -8,6 +8,7 @@
 , cargoLockContentsList ? [ ]
 , cargoLockList ? [ ]
 , cargoLockParsedList ? [ ]
+, outputHashes ? { }
 }@args:
 let
   inherit (builtins)
@@ -57,7 +58,7 @@ let
   } // optionalAttrs (args ? registries) { inherit (args) registries; });
 
   vendoredGit = vendorGitDeps {
-    inherit lockPackages;
+    inherit lockPackages outputHashes;
   };
 
   linkSources = sources: concatMapStrings