Git hashing as a new file ingestion method --- contains #3754

perl/lib/Nix/Store.xs

@@ -57,7 +57,8 @@ void setVerbosity(int level)
 int isValidPath(char * path)
     CODE:
         try {
-            RETVAL = store()->isValidPath(store()->parseStorePath(path));
+            auto storePath = store()->parseStorePath(path);
+            RETVAL = store()->isValidPath(storePath);
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -68,7 +69,8 @@ int isValidPath(char * path)
 SV * queryReferences(char * path)
     PPCODE:
         try {
-            for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references)
+            auto storePath = store()->parseStorePath(path);
+            for (auto & i : store()->queryPathInfo(storePath)->referencesPossiblyToSelf())
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -78,7 +80,8 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
+            auto storePath = store()->parseStorePath(path);
+            auto s = store()->queryPathInfo(storePath)->narHash.to_string(Base32, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -88,7 +91,8 @@ SV * queryPathHash(char * path)
 SV * queryDeriver(char * path)
     PPCODE:
         try {
-            auto info = store()->queryPathInfo(store()->parseStorePath(path));
+            auto storePath = store()->parseStorePath(path);
+            auto info = store()->queryPathInfo(storePath);
             if (!info->deriver) XSRETURN_UNDEF;
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
         } catch (Error & e) {
@@ -99,7 +103,8 @@ SV * queryDeriver(char * path)
 SV * queryPathInfo(char * path, int base32)
     PPCODE:
         try {
-            auto info = store()->queryPathInfo(store()->parseStorePath(path));
+            auto storePath = store()->parseStorePath(path);
+            auto info = store()->queryPathInfo(storePath);
             if (!info->deriver)
                 XPUSHs(&PL_sv_undef);
             else
@@ -109,7 +114,7 @@ SV * queryPathInfo(char * path, int base32)
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
             AV * refs = newAV();
-            for (auto & i : info->references)
+            for (auto & i : info->referencesPossiblyToSelf())
                 av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0));
             XPUSHs(sv_2mortal(newRV((SV *) refs)));
             AV * sigs = newAV();

src/libcmd/command.cc

@@ -52,7 +52,8 @@ ref<Store> StoreCommand::createStore()
 
 void StoreCommand::run()
 {
-    run(getStore());
+    auto store = getStore();
+    run(store);
 }
 
 CopyCommand::CopyCommand()

src/libcmd/common-eval-args.cc

@@ -167,15 +167,17 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 SourcePath lookupFileArg(EvalState & state, std::string_view s)
 {
     if (EvalSettings::isPseudoUrl(s)) {
-        auto storePath = fetchers::downloadTarball(
-            state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath;
+        auto storePath = state.store->makeFixedOutputPathFromCA(
+            fetchers::downloadTarball(
+                state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath);
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
-        auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath;
+        auto storePath = state.store->makeFixedOutputPathFromCA(
+            flakeRef.resolve(state.store).fetchTree(state.store).first.storePath);
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 

src/libcmd/installables.cc

@@ -403,7 +403,7 @@ ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake)
 {
-    auto fingerprint = lockedFlake->getFingerprint();
+    auto fingerprint = lockedFlake->getFingerprint(*state.store);
     return make_ref<nix::eval_cache::EvalCache>(
         evalSettings.useEvalCache && evalSettings.pureEval
             ? std::optional { std::cref(fingerprint) }

src/libexpr/flake/flake.cc

@@ -63,10 +63,12 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
 
     auto [tree, lockedRef] = *fetched;
 
+    auto storePath = state.store->makeFixedOutputPathFromCA(tree.storePath);
+
     debug("got tree '%s' from '%s'",
-        state.store->printStorePath(tree.storePath), lockedRef);
+        state.store->printStorePath(storePath), lockedRef);
 
-    state.allowPath(tree.storePath);
+    state.allowPath(storePath);
 
     assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
 
@@ -210,7 +212,8 @@ static Flake getFlake(
     auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
     if (!isInDir(flakeFile, sourceInfo.actualPath))
         throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
-            lockedRef, state.store->printStorePath(sourceInfo.storePath));
+            lockedRef,
+            state.store->printStorePath(state.store->makeFixedOutputPathFromCA(sourceInfo.storePath)));
 
     Flake flake {
         .originalRef = originalRef,
@@ -891,14 +894,14 @@ static RegisterPrimOp r4({
 
 }
 
-Fingerprint LockedFlake::getFingerprint() const
+Fingerprint LockedFlake::getFingerprint(const Store & store) const
 {
     // FIXME: as an optimization, if the flake contains a lock file
     // and we haven't changed it, then it's sufficient to use
     // flake.sourceInfo.storePath for the fingerprint.
     return hashString(htSHA256,
         fmt("%s;%s;%d;%d;%s",
-            flake.sourceInfo->storePath.to_string(),
+            store.makeFixedOutputPathFromCA(flake.sourceInfo->storePath).to_string(),
             flake.lockedRef.subdir,
             flake.lockedRef.input.getRevCount().value_or(0),
             flake.lockedRef.input.getLastModified().value_or(0),

src/libexpr/flake/flake.hh

@@ -105,7 +105,7 @@ struct LockedFlake
     Flake flake;
     LockFile lockFile;
 
-    Fingerprint getFingerprint() const;
+    Fingerprint getFingerprint(const Store & store) const;
 };
 
 struct LockFlags

src/libexpr/flake/lockfile.cc

@@ -40,7 +40,7 @@ LockedNode::LockedNode(const nlohmann::json & json)
             fetchers::attrsToJSON(lockedRef.input.toAttrs()));
 }
 
-StorePath LockedNode::computeStorePath(Store & store) const
+StorePathDescriptor LockedNode::computeStorePath(Store & store) const
 {
     return lockedRef.input.computeStorePath(store);
 }

src/libexpr/flake/lockfile.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "flakeref.hh"
+#include "content-address.hh"
 
 #include <nlohmann/json_fwd.hpp>
 
@@ -47,7 +48,7 @@ struct LockedNode : Node
 
     LockedNode(const nlohmann::json & json);
 
-    StorePath computeStorePath(Store & store) const;
+    StorePathDescriptor computeStorePath(Store & store) const;
 };
 
 struct LockFile

src/libexpr/parser.y

@@ -781,8 +781,12 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
 
     if (EvalSettings::isPseudoUrl(value)) {
         try {
-            auto storePath = fetchers::downloadTarball(
-                store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath;
+            auto storePath = store->makeFixedOutputPathFromCA(
+                fetchers::downloadTarball(
+                    store,
+                    EvalSettings::resolvePseudoUrl(value),
+                    "source",
+                    false).tree.storePath);
             res = { store->toRealPath(storePath) };
         } catch (FileTransferError & e) {
             logWarning({
@@ -796,7 +800,8 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
         debug("fetching flake search path element '%s''", value);
-        auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath;
+        auto storePath = store->makeFixedOutputPathFromCA(
+    		flakeRef.resolve(store).fetchTree(store).first.storePath);
         res = { store->toRealPath(storePath) };
     }
 

src/libexpr/primops.cc

@@ -1618,8 +1618,9 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V
         state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path));
     StorePathSet refs;
     if (state.store->isInStore(path.path.abs())) {
+        auto p = state.store->toStorePath(path.path.abs()).first;
         try {
-            refs = state.store->queryPathInfo(state.store->toStorePath(path.path.abs()).first)->references;
+            refs = state.store->queryPathInfo(p)->referencesPossiblyToSelf();
         } catch (Error &) { // FIXME: should be InvalidPathError
         }
         // Re-scan references to filter down to just the ones that actually occur in the file.
@@ -2188,7 +2189,7 @@ static void addPath(
             try {
                 auto [storePath, subPath] = state.store->toStorePath(path);
                 // FIXME: we should scanForReferences on the path before adding it
-                refs = state.store->queryPathInfo(storePath)->references;
+                refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf();
                 path = state.store->toRealPath(storePath) + subPath;
             } catch (Error &) { // FIXME: should be InvalidPathError
             }

src/libexpr/primops/context.cc

@@ -187,8 +187,9 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
                 .errPos = state.positions[i.pos]
             });
         auto namePath = state.store->parseStorePath(name);
-        if (!settings.readOnlyMode)
+        if (!settings.readOnlyMode) {
             state.store->ensurePath(namePath);
+        }
         state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context");
         auto iter = i.value->attrs->find(sPath);
         if (iter != i.value->attrs->end()) {

src/libexpr/primops/fetchMercurial.cc

@@ -74,7 +74,8 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
     auto [tree, input2] = input.fetch(state.store);
 
     auto attrs2 = state.buildBindings(8);
-    state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath));
+    auto storePath = state.store->makeFixedOutputPathFromCA(tree.storePath);
+    state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath));
     if (input2.getRef())
         attrs2.alloc("branch").mkString(*input2.getRef());
     // Backward compatibility: set 'rev' to
@@ -86,7 +87,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
         attrs2.alloc("revCount").mkInt(*revCount);
     v.mkAttrs(attrs2);
 
-    state.allowPath(tree.storePath);
+    state.allowPath(state.store->makeFixedOutputPathFromCA(tree.storePath));
 }
 
 static RegisterPrimOp r_fetchMercurial({

src/libexpr/primops/fetchTree.cc

@@ -25,14 +25,19 @@ void emitTreeAttrs(
 
     auto attrs = state.buildBindings(10);
 
+    auto storePath = state.store->makeFixedOutputPathFromCA(tree.storePath);
 
-    state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath));
+    state.mkStorePathString(storePath, attrs.alloc(state.sOutPath));
 
     // FIXME: support arbitrary input attributes.
 
-    auto narHash = input.getNarHash();
-    assert(narHash);
-    attrs.alloc("narHash").mkString(narHash->to_string(SRI, true));
+    if (auto narHash = input.getNarHash()) {
+        attrs.alloc("narHash").mkString(narHash->to_string(SRI, true));
+    } else if (auto treeHash = input.getTreeHash()) {
+        attrs.alloc("treeHash").mkString(treeHash->to_string(SRI, true));
+    } else
+        /* Must have either tree hash or NAR hash */
+        assert(false);
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(
@@ -50,6 +55,11 @@ void emitTreeAttrs(
             attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
 
+        if (auto treeHash = input.getTreeHash()) {
+            attrs.alloc("treeHash").mkString(treeHash->gitRev());
+            attrs.alloc("shortTreeHash").mkString(treeHash->gitRev());
+        }
+
         if (auto revCount = input.getRevCount())
             attrs.alloc("revCount").mkInt(*revCount);
         else if (emptyRevFallback)
@@ -188,7 +198,7 @@ static void fetchTree(
 
     auto [tree, input2] = input.fetch(state.store);
 
-    state.allowPath(tree.storePath);
+    state.allowPath(state.store->makeFixedOutputPathFromCA(tree.storePath));
 
     emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false);
 }
@@ -250,12 +260,14 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     if (evalSettings.pureEval && !expectedHash)
         state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who));
 
-    // early exit if pinned and already in the store
-    if (expectedHash && expectedHash->type == htSHA256) {
+    // early exit if pinned and already in the store, or substituted successfully
+    if (expectedHash) {
+        auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+
         auto expectedPath = state.store->makeFixedOutputPath(
             name,
             FixedOutputInfo {
-                .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
+                .method = method,
                 .hash = *expectedHash,
                 .references = {}
             });
@@ -264,6 +276,15 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
             state.allowAndSetStorePathString(expectedPath, v);
             return;
         }
+
+        // try to substitute if we can
+
+        auto substitutableStorePath = fetchers::trySubstitute(state.store, method, *expectedHash, name);
+
+        if (substitutableStorePath) {
+            state.allowAndSetStorePathString(*substitutableStorePath, v);
+            return;
+        }
     }
 
     // TODO: fetching may fail, yet the path may be substitutable.
@@ -273,16 +294,18 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
         ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath
         : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
 
+    auto actualStorePath = state.store->makeFixedOutputPathFromCA(storePath);
+
     if (expectedHash) {
         auto hash = unpack
             ? state.store->queryPathInfo(storePath)->narHash
-            : hashFile(htSHA256, state.store->toRealPath(storePath));
+            : hashFile(htSHA256, state.store->toRealPath(actualStorePath));
         if (hash != *expectedHash)
             state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  specified: %s\n  got:       %s",
                 *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)));
     }
 
-    state.allowAndSetStorePathString(storePath, v);
+    state.allowAndSetStorePathString(actualStorePath, v);
 }
 
 static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, Value & v)

src/libfetchers/cache.cc

@@ -51,18 +51,19 @@ struct CacheImpl : Cache
         ref<Store> store,
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
-        const StorePath & storePath,
+        const StorePathDescriptor & storePathDesc,
         bool locked) override
     {
         _state.lock()->add.use()
             (attrsToJSON(inAttrs).dump())
             (attrsToJSON(infoAttrs).dump())
-            (store->printStorePath(storePath))
+            // FIXME should use JSON for store path descriptor
+            (renderStorePathDescriptor(storePathDesc))
             (locked)
             (time(0)).exec();
     }
 
-    std::optional<std::pair<Attrs, StorePath>> lookup(
+    std::optional<std::pair<Attrs, StorePathDescriptor>> lookup(
         ref<Store> store,
         const Attrs & inAttrs) override
     {
@@ -90,9 +91,10 @@ struct CacheImpl : Cache
         }
 
         auto infoJSON = stmt.getStr(0);
-        auto storePath = store->parseStorePath(stmt.getStr(1));
+        auto storePathDesc = parseStorePathDescriptor(stmt.getStr(1));
         auto locked = stmt.getInt(2) != 0;
         auto timestamp = stmt.getInt(3);
+        auto storePath = store->makeFixedOutputPathFromCA(storePathDesc);
 
         store->addTempRoot(storePath);
         if (!store->isValidPath(storePath)) {
@@ -107,7 +109,7 @@ struct CacheImpl : Cache
         return Result {
             .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
             .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
-            .storePath = std::move(storePath)
+            .storePath = std::move(storePathDesc)
         };
     }
 };