Git hashing for Git Fetching

Now that we have Git hashing of (the FSOs of) store objects, we should
teach the git fetcher how to use it. It can make especially good use of
it because then we can use a single hash for both identifying what to
fetch, and validating the download.

Part of RFC 133

Extracted from our old IPFS branches.

Co-Authored-By: Matthew Bauer <mjbauer95@gmail.com>
Co-Authored-By: Carlo Nucera <carlo.nucera@protonmail.com>
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>

src/libexpr/primops/fetchTree.cc

@@ -31,9 +31,13 @@ void emitTreeAttrs(
 
     // FIXME: support arbitrary input attributes.
 
-    auto narHash = input.getNarHash();
-    assert(narHash);
-    attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
+    if (auto narHash = input.getNarHash()) {
+        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
+    } else if (auto treeHash = input.getTreeHash()) {
+        attrs.alloc("treeHash").mkString(treeHash->to_string(HashFormat::SRI, true));
+    } else
+        /* Must have either tree hash or NAR hash */
+        assert(false);
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(
@@ -51,6 +55,10 @@ void emitTreeAttrs(
             attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
 
+        if (auto treeHash = input.getTreeHash()) {
+            attrs.alloc("treeHash").mkString(treeHash->gitRev());
+        }
+
         if (auto revCount = input.getRevCount())
             attrs.alloc("revCount").mkInt(*revCount);
         else if (emptyRevFallback)

src/libfetchers/fetchers.cc

@@ -301,14 +301,19 @@ std::string Input::getName() const
 
 StorePath Input::computeStorePath(Store & store) const
 {
-    auto narHash = getNarHash();
-    if (!narHash)
-        throw Error("cannot compute store path for unlocked input '%s'", to_string());
-    return store.makeFixedOutputPath(getName(), FixedOutputInfo {
-        .method = FileIngestionMethod::Recursive,
-        .hash = *narHash,
-        .references = {},
-    });
+    if (auto treeHash = getTreeHash())
+        return store.makeFixedOutputPath(getName(), FixedOutputInfo {
+            .method = FileIngestionMethod::Git,
+            .hash = *treeHash,
+            .references = {},
+        });
+    if (auto narHash = getNarHash())
+        return store.makeFixedOutputPath(getName(), FixedOutputInfo {
+            .method = FileIngestionMethod::Recursive,
+            .hash = *narHash,
+            .references = {},
+        });
+    throw Error("cannot compute store path for unlocked input '%s'", to_string());
 }
 
 std::string Input::getType() const
@@ -351,6 +356,15 @@ std::optional<Hash> Input::getRev() const
     return hash;
 }
 
+std::optional<Hash> Input::getTreeHash() const
+{
+    if (auto s = maybeGetStrAttr(attrs, "treeHash")) {
+        experimentalFeatureSettings.require(Xp::GitHashing);
+        return Hash::parseAny(*s, HashAlgorithm::SHA1);
+    }
+    return {};
+}
+
 std::optional<uint64_t> Input::getRevCount() const
 {
     if (auto n = maybeGetIntAttr(attrs, "revCount"))

src/libfetchers/fetchers.hh

@@ -121,6 +121,7 @@ public:
     std::optional<Hash> getNarHash() const;
     std::optional<std::string> getRef() const;
     std::optional<Hash> getRev() const;
+    std::optional<Hash> getTreeHash() const;
     std::optional<uint64_t> getRevCount() const;
     std::optional<time_t> getLastModified() const;
 

src/libfetchers/git-utils.cc

@@ -57,7 +57,7 @@ bool operator == (const git_oid & oid1, const git_oid & oid2)
 
 namespace nix {
 
-struct GitInputAccessor;
+struct GitInputAccessorImpl;
 
 // Some wrapper types that ensure that the git_*_free functions get called.
 template<auto del>
@@ -334,9 +334,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     }
 
     /**
-     * A 'GitInputAccessor' with no regard for export-ignore or any other transformations.
+     * A 'GitInputAccessorImpl' with no regard for export-ignore or any other transformations.
      */
-    ref<GitInputAccessor> getRawAccessor(const Hash & rev);
+    ref<GitInputAccessorImpl> getRawAccessor(const Hash & rev);
+
+    ref<GitInputAccessor> getPlainAccessor(const Hash & rev) override;
 
     ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) override;
 
@@ -477,17 +479,24 @@ ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create,
 /**
  * Raw git tree input accessor.
  */
-struct GitInputAccessor : InputAccessor
+struct GitInputAccessorImpl : GitInputAccessor
 {
     ref<GitRepoImpl> repo;
     Tree root;
 
-    GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
+    GitInputAccessorImpl(ref<GitRepoImpl> repo_, const Hash & rev)
         : repo(repo_)
         , root(peelObject<Tree>(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE))
     {
     }
 
+    Hash getTreeHash() override
+    {
+        auto * oid = git_tree_id(root.get());
+        assert(oid);
+        return toHash(*oid);
+    }
+
     std::string readBlob(const CanonPath & path, bool symlink)
     {
         auto blob = getBlob(path, symlink);
@@ -922,17 +931,22 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
     }
 };
 
-ref<GitInputAccessor> GitRepoImpl::getRawAccessor(const Hash & rev)
+ref<GitInputAccessorImpl> GitRepoImpl::getRawAccessor(const Hash & rev)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    return make_ref<GitInputAccessor>(self, rev);
+    return make_ref<GitInputAccessorImpl>(self, rev);
+}
+
+ref<GitInputAccessor> GitRepoImpl::getPlainAccessor(const Hash & rev)
+{
+    return getRawAccessor(rev);
 }
 
 ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
 {
-    auto self = ref<GitRepoImpl>(shared_from_this());
-    ref<GitInputAccessor> rawGitAccessor = getRawAccessor(rev);
+    ref<GitInputAccessorImpl> rawGitAccessor = getRawAccessor(rev);
     if (exportIgnore) {
+        auto self = ref<GitRepoImpl>(shared_from_this());
         return make_ref<GitExportIgnoreInputAccessor>(self, rawGitAccessor, rev);
     }
     else {

src/libfetchers/git-utils.hh

@@ -16,6 +16,16 @@ struct GitFileSystemObjectSink : FileSystemObjectSink
     virtual Hash sync() = 0;
 };
 
+/**
+ * Git Input Accessor
+ *
+ * Created from `GitRepo`. Support some additional operations.
+ */
+struct GitInputAccessor : InputAccessor
+{
+    virtual Hash getTreeHash() = 0;
+};
+
 struct GitRepo
 {
     virtual ~GitRepo()
@@ -75,6 +85,8 @@ struct GitRepo
 
     virtual bool hasObject(const Hash & oid) = 0;
 
+    virtual ref<GitInputAccessor> getPlainAccessor(const Hash & rev) = 0;
+
     virtual ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) = 0;
 
     virtual ref<InputAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;