Merge pull request #25 from obsidiansystems/ipfs-git-ipld

Support substituting in fetchGit
obsidiansystems · Jun 23, 2020 · 0de3063 · 0de3063
2 parents f85fc65 + 4074142
commit 0de3063
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 3 deletions.
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
@@ -134,6 +134,20 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     if (evalSettings.pureEval && !expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
+    // try to substitute if we can
+    if (settings.useSubstitutes && expectedHash) {
+        auto substitutableStorePath = fetchers::trySubstitute(state.store,
+            unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, *expectedHash, name);
+        if (substitutableStorePath) {
+            auto substitutablePath = state.store->toRealPath(*substitutableStorePath);
+            if (state.allowedPaths)
+                state.allowedPaths->insert(substitutablePath);
+
+            mkString(v, substitutablePath, PathSet({substitutablePath}));
+            return;
+        }
+    }
+
     auto storePath =
         unpack
         ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
@@ -72,4 +72,22 @@ std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store)
     return {std::move(tree), input};
 }
 
+std::optional<StorePath> trySubstitute(ref<Store> store, FileIngestionMethod ingestionMethod,
+    Hash hash, std::string_view name)
+{
+    auto substitutablePath = store->makeFixedOutputPath(ingestionMethod, hash, name);
+
+    try {
+        store->ensurePath(substitutablePath);
+
+        debug("using substituted path '%s'", store->printStorePath(substitutablePath));
+
+        return substitutablePath;
+    } catch (Error & e) {
+        debug("substitution of path '%s' failed: %s", store->printStorePath(substitutablePath), e.what());
+    }
+
+    return std::nullopt;
+}
+
 }
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
@@ -100,4 +100,7 @@ Tree downloadTarball(
     const std::string & name,
     bool immutable);
 
+std::optional<StorePath> trySubstitute(ref<Store> store, FileIngestionMethod ingestionMethod,
+    Hash hash, std::string_view name);
+
 }
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
@@ -106,6 +106,24 @@ struct GitInput : Input
 
         auto ingestionMethod = treeHash ? FileIngestionMethod::Git : FileIngestionMethod::Recursive;
 
+        // try to substitute
+        if (settings.useSubstitutes && treeHash && !submodules) {
+            auto storePath = fetchers::trySubstitute(store, ingestionMethod, *treeHash, name);
+            if (storePath) {
+                return {
+                    Tree {
+                        .actualPath = store->toRealPath(*storePath),
+                        .storePath = std::move(*storePath),
+                        .info = TreeInfo {
+                            .revCount = std::nullopt,
+                            .lastModified = 0,
+                        },
+                    },
+                    input
+                };
+            }
+        }
+
         std::string cacheType = "git";
         if (shallow) cacheType += "-shallow";
         if (submodules) cacheType += "-submodules";
@@ -385,7 +403,7 @@ struct GitInput : Input
             unpackTarfile(*source, tmpDir);
         }
 
-        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, ingestionMethod == FileIngestionMethod::Git ? htSHA1 : htSHA256, filter);
+        auto storePath = store->addToStore(name, tmpDir, ingestionMethod, ingestionMethod == FileIngestionMethod::Git ? htSHA1 : htSHA256, filter);
 
         // verify treeHash is what we actually obtained in the nix store
         if (input->treeHash) {

diff --git a/tests/git.sh b/tests/git.sh
@@ -1,6 +1,7 @@
 source common.sh
 
 clearStore
+clearCache
 
 try () {
     hash=$(nix hash-git --base16 --type sha1 $TEST_ROOT/hash-path)
@@ -40,8 +41,6 @@ test "$hash3" = "sha256:1i2x80840igikhbyy7nqf08ymx3a6n83x1fzyrxvddf0sdl5nqvp"
 if [[ -n $(type -p git) ]]; then
     repo=$TEST_ROOT/git
 
-    export _NIX_FORCE_HTTP=1
-
     rm -rf $repo $TEST_HOME/.cache/nix
 
     git init $repo
@@ -64,6 +63,12 @@ if [[ -n $(type -p git) ]]; then
 
     # Submodules cause error.
     (! nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file://$repo; treeHash = \"$treeHash\"; submodules = true; }).outPath")
+
+    # Check that we can substitute it from other places.
+    nix copy --to file://$cacheDir $path
+    nix-store --delete $path
+    path2=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$treeHash\"; }).outPath" --substituters file://$cacheDir --option substitute true)
+    [ $path2 = $path ]
 else
     echo "Git not installed; skipping Git tests"
 fi