diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index c38ea2d2b80..4dd3f67bb31 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -57,7 +57,8 @@ void setVerbosity(int level) int isValidPath(char * path) CODE: try { - RETVAL = store()->isValidPath(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + RETVAL = store()->isValidPath(storePath); } catch (Error & e) { croak("%s", e.what()); } @@ -68,7 +69,8 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) + auto storePath = store()->parseStorePath(path); + for (auto & i : store()->queryPathInfo(storePath)->referencesPossiblyToSelf()) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -78,7 +80,8 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto storePath = store()->parseStorePath(path); + auto s = store()->queryPathInfo(storePath)->narHash.to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -88,7 +91,8 @@ SV * queryPathHash(char * path) SV * queryDeriver(char * path) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + auto info = store()->queryPathInfo(storePath); if (!info->deriver) XSRETURN_UNDEF; XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); } catch (Error & e) { @@ -99,7 +103,8 @@ SV * queryDeriver(char * path) SV * queryPathInfo(char * path, int base32) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + auto info = store()->queryPathInfo(storePath); if (!info->deriver) XPUSHs(&PL_sv_undef); else @@ -109,7 +114,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 4fc1979563d..813d3d46252 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -52,7 +52,8 @@ ref StoreCommand::createStore() void StoreCommand::run() { - run(getStore()); + auto store = getStore(); + run(store); } CopyCommand::CopyCommand() diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e36bda52fd9..de9dcb815ac 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -167,15 +167,17 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) SourcePath lookupFileArg(EvalState & state, std::string_view s) { if (EvalSettings::isPseudoUrl(s)) { - auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath; + auto storePath = state.store->makeFixedOutputPathFromCA( + fetchers::downloadTarball( + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath); return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); - auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; + auto storePath = state.store->makeFixedOutputPathFromCA( + flakeRef.resolve(state.store).fetchTree(state.store).first.storePath); return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index eb19030847d..671dceecfe2 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -403,7 +403,7 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); + auto fingerprint = lockedFlake->getFingerprint(*state.store); return make_ref( evalSettings.useEvalCache && evalSettings.pureEval ? std::optional { std::cref(fingerprint) } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 6a27ea2e8dc..54c17ae2cfc 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -63,10 +63,12 @@ static std::tuple fetchOrSubstituteTree( auto [tree, lockedRef] = *fetched; + auto storePath = state.store->makeFixedOutputPathFromCA(tree.storePath); + debug("got tree '%s' from '%s'", - state.store->printStorePath(tree.storePath), lockedRef); + state.store->printStorePath(storePath), lockedRef); - state.allowPath(tree.storePath); + state.allowPath(storePath); assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); @@ -210,7 +212,8 @@ static Flake getFlake( auto flakeFile = canonPath(flakeDir + "/flake.nix", true); if (!isInDir(flakeFile, sourceInfo.actualPath)) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", - lockedRef, state.store->printStorePath(sourceInfo.storePath)); + lockedRef, + state.store->printStorePath(state.store->makeFixedOutputPathFromCA(sourceInfo.storePath))); Flake flake { .originalRef = originalRef, @@ -891,14 +894,14 @@ static RegisterPrimOp r4({ } -Fingerprint LockedFlake::getFingerprint() const +Fingerprint LockedFlake::getFingerprint(const Store & store) const { // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, fmt("%s;%s;%d;%d;%s", - flake.sourceInfo->storePath.to_string(), + store.makeFixedOutputPathFromCA(flake.sourceInfo->storePath).to_string(), flake.lockedRef.subdir, flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getLastModified().value_or(0), diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index c1d1b71e509..9dd7b05741a 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -105,7 +105,7 @@ struct LockedFlake Flake flake; LockFile lockFile; - Fingerprint getFingerprint() const; + Fingerprint getFingerprint(const Store & store) const; }; struct LockFlags diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 3c202967a02..96e76647a5c 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -40,7 +40,7 @@ LockedNode::LockedNode(const nlohmann::json & json) fetchers::attrsToJSON(lockedRef.input.toAttrs())); } -StorePath LockedNode::computeStorePath(Store & store) const +StorePathDescriptor LockedNode::computeStorePath(Store & store) const { return lockedRef.input.computeStorePath(store); } diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index ba4c0c8485c..716ddced752 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -2,6 +2,7 @@ ///@file #include "flakeref.hh" +#include "content-address.hh" #include @@ -47,7 +48,7 @@ struct LockedNode : Node LockedNode(const nlohmann::json & json); - StorePath computeStorePath(Store & store) const; + StorePathDescriptor computeStorePath(Store & store) const; }; struct LockFile diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 792f51fde32..33b48598db6 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -781,8 +781,12 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa if (EvalSettings::isPseudoUrl(value)) { try { - auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + auto storePath = store->makeFixedOutputPathFromCA( + fetchers::downloadTarball( + store, + EvalSettings::resolvePseudoUrl(value), + "source", + false).tree.storePath); res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ @@ -796,7 +800,8 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); debug("fetching flake search path element '%s''", value); - auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; + auto storePath = store->makeFixedOutputPathFromCA( + flakeRef.resolve(store).fetchTree(store).first.storePath); res = { store->toRealPath(storePath) }; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e2b1ac4f65e..68c19df3350 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1618,8 +1618,9 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path)); StorePathSet refs; if (state.store->isInStore(path.path.abs())) { + auto p = state.store->toStorePath(path.path.abs()).first; try { - refs = state.store->queryPathInfo(state.store->toStorePath(path.path.abs()).first)->references; + refs = state.store->queryPathInfo(p)->referencesPossiblyToSelf(); } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -2188,7 +2189,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->references; + refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index e8542503a42..2c99eb08412 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -187,8 +187,9 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar .errPos = state.positions[i.pos] }); auto namePath = state.store->parseStorePath(name); - if (!settings.readOnlyMode) + if (!settings.readOnlyMode) { state.store->ensurePath(namePath); + } state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index b9ff01c16e1..f8817b4b94d 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -74,7 +74,8 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto [tree, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); + auto storePath = state.store->makeFixedOutputPathFromCA(tree.storePath); + state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to @@ -86,7 +87,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("revCount").mkInt(*revCount); v.mkAttrs(attrs2); - state.allowPath(tree.storePath); + state.allowPath(state.store->makeFixedOutputPathFromCA(tree.storePath)); } static RegisterPrimOp r_fetchMercurial({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f040a35109a..5f5cb2ff130 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -25,14 +25,19 @@ void emitTreeAttrs( auto attrs = state.buildBindings(10); + auto storePath = state.store->makeFixedOutputPathFromCA(tree.storePath); - state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. - auto narHash = input.getNarHash(); - assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); + if (auto narHash = input.getNarHash()) { + attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); + } else if (auto treeHash = input.getTreeHash()) { + attrs.alloc("treeHash").mkString(treeHash->to_string(SRI, true)); + } else + /* Must have either tree hash or NAR hash */ + assert(false); if (input.getType() == "git") attrs.alloc("submodules").mkBool( @@ -50,6 +55,11 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(emptyHash.gitShortRev()); } + if (auto treeHash = input.getTreeHash()) { + attrs.alloc("treeHash").mkString(treeHash->gitRev()); + attrs.alloc("shortTreeHash").mkString(treeHash->gitRev()); + } + if (auto revCount = input.getRevCount()) attrs.alloc("revCount").mkInt(*revCount); else if (emptyRevFallback) @@ -188,7 +198,7 @@ static void fetchTree( auto [tree, input2] = input.fetch(state.store); - state.allowPath(tree.storePath); + state.allowPath(state.store->makeFixedOutputPathFromCA(tree.storePath)); emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); } @@ -250,12 +260,14 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (evalSettings.pureEval && !expectedHash) state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who)); - // early exit if pinned and already in the store - if (expectedHash && expectedHash->type == htSHA256) { + // early exit if pinned and already in the store, or substituted successfully + if (expectedHash) { + auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .method = method, .hash = *expectedHash, .references = {} }); @@ -264,6 +276,15 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.allowAndSetStorePathString(expectedPath, v); return; } + + // try to substitute if we can + + auto substitutableStorePath = fetchers::trySubstitute(state.store, method, *expectedHash, name); + + if (substitutableStorePath) { + state.allowAndSetStorePathString(*substitutableStorePath, v); + return; + } } // TODO: fetching may fail, yet the path may be substitutable. @@ -273,16 +294,18 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + auto actualStorePath = state.store->makeFixedOutputPathFromCA(storePath); + if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, state.store->toRealPath(storePath)); + : hashFile(htSHA256, state.store->toRealPath(actualStorePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true))); } - state.allowAndSetStorePathString(storePath, v); + state.allowAndSetStorePathString(actualStorePath, v); } static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, Value & v) diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 0c8ecac9d48..2a55f085fde 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -51,18 +51,19 @@ struct CacheImpl : Cache ref store, const Attrs & inAttrs, const Attrs & infoAttrs, - const StorePath & storePath, + const StorePathDescriptor & storePathDesc, bool locked) override { _state.lock()->add.use() (attrsToJSON(inAttrs).dump()) (attrsToJSON(infoAttrs).dump()) - (store->printStorePath(storePath)) + // FIXME should use JSON for store path descriptor + (renderStorePathDescriptor(storePathDesc)) (locked) (time(0)).exec(); } - std::optional> lookup( + std::optional> lookup( ref store, const Attrs & inAttrs) override { @@ -90,9 +91,10 @@ struct CacheImpl : Cache } auto infoJSON = stmt.getStr(0); - auto storePath = store->parseStorePath(stmt.getStr(1)); + auto storePathDesc = parseStorePathDescriptor(stmt.getStr(1)); auto locked = stmt.getInt(2) != 0; auto timestamp = stmt.getInt(3); + auto storePath = store->makeFixedOutputPathFromCA(storePathDesc); store->addTempRoot(storePath); if (!store->isValidPath(storePath)) { @@ -107,7 +109,7 @@ struct CacheImpl : Cache return Result { .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)), - .storePath = std::move(storePath) + .storePath = std::move(storePathDesc) }; } }; diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index ae398d0404b..c92792843be 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -13,10 +13,10 @@ struct Cache ref store, const Attrs & inAttrs, const Attrs & infoAttrs, - const StorePath & storePath, + const StorePathDescriptor & storePath, bool locked) = 0; - virtual std::optional> lookup( + virtual std::optional> lookup( ref store, const Attrs & inAttrs) = 0; @@ -24,7 +24,7 @@ struct Cache { bool expired = false; Attrs infoAttrs; - StorePath storePath; + StorePathDescriptor storePath; }; virtual std::optional lookupExpired( diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e683b9f804a..6834f12627d 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "store-api.hh" +#include "archive.hh" #include @@ -25,6 +26,8 @@ static void fixupInput(Input & input) input.getRef(); if (input.getRev()) input.locked = true; + if (input.getTreeHash()) + input.locked = true; input.getRevCount(); input.getLastModified(); if (input.getNarHash()) @@ -89,7 +92,7 @@ Attrs Input::toAttrs() const bool Input::hasAllInfo() const { - return getNarHash() && scheme && scheme->hasAllInfo(*this); + return scheme && scheme->hasAllInfo(*this); } bool Input::operator ==(const Input & other) const @@ -117,20 +120,28 @@ std::pair Input::fetch(ref store) const original source). So check that. */ if (hasAllInfo()) { try { - auto storePath = computeStorePath(*store); + auto storePathDesc = computeStorePath(*store); + + store->ensurePath(std::cref(storePathDesc)); - store->ensurePath(storePath); + auto storePath = store->makeFixedOutputPathFromCA(storePathDesc); debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); - return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this}; + return { + Tree { + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePathDesc), + }, + *this, + }; } catch (Error & e) { debug("substitution of input '%s' failed: %s", to_string(), e.what()); } } - auto [storePath, input] = [&]() -> std::pair { + auto [storePath, input] = [&]() -> std::pair { try { return scheme->fetch(store, *this); } catch (Error & e) { @@ -140,7 +151,7 @@ std::pair Input::fetch(ref store) const }(); Tree tree { - .actualPath = store->toRealPath(storePath), + .actualPath = store->toRealPath(store->makeFixedOutputPathFromCA(storePath)), .storePath = storePath, }; @@ -211,16 +222,27 @@ std::string Input::getName() const return maybeGetStrAttr(attrs, "name").value_or("source"); } -StorePath Input::computeStorePath(Store & store) const +StorePathDescriptor Input::computeStorePath(Store & store) const { - auto narHash = getNarHash(); - if (!narHash) - throw Error("cannot compute store path for unlocked input '%s'", to_string()); - return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .method = FileIngestionMethod::Recursive, - .hash = *narHash, - .references = {}, - }); + if (auto treeHash = getTreeHash()) + return StorePathDescriptor { + getName(), + FixedOutputInfo { + .method = FileIngestionMethod::Git, + .hash = *treeHash, + .references = {}, + }, + }; + if (auto narHash = getNarHash()) + return StorePathDescriptor { + "source", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + .references = {}, + }, + }; + throw Error("cannot compute store path for unlocked input '%s'", to_string()); } std::string Input::getType() const @@ -262,6 +284,15 @@ std::optional Input::getRev() const return hash; } +std::optional Input::getTreeHash() const +{ + if (auto s = maybeGetStrAttr(attrs, "treeHash")) { + experimentalFeatureSettings.require(Xp::GitHashing); + return Hash::parseAny(*s, htSHA1); + } + return {}; +} + std::optional Input::getRevCount() const { if (auto n = maybeGetIntAttr(attrs, "revCount")) @@ -308,4 +339,30 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name) +{ + auto ca = StorePathDescriptor { + .name = std::string { name }, + .info = FixedOutputInfo { + ingestionMethod, + hash, + {} + }, + }; + auto substitutablePath = store->makeFixedOutputPathFromCA(ca); + + try { + store->ensurePath(ca); + + debug("using substituted path '%s'", store->printStorePath(substitutablePath)); + + return substitutablePath; + } catch (Error & e) { + debug("substitution of path '%s' failed: %s", store->printStorePath(substitutablePath), e.what()); + } + + return std::nullopt; +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 6e10e95134f..a52311bcf48 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -6,6 +6,7 @@ #include "path.hh" #include "attrs.hh" #include "url.hh" +#include "content-address.hh" #include @@ -16,7 +17,7 @@ namespace nix::fetchers { struct Tree { Path actualPath; - StorePath storePath; + StorePathDescriptor storePath; }; struct InputScheme; @@ -103,13 +104,14 @@ public: std::string getName() const; - StorePath computeStorePath(Store & store) const; + StorePathDescriptor computeStorePath(Store & store) const; // Convenience functions for common attributes. std::string getType() const; std::optional getNarHash() const; std::optional getRef() const; std::optional getRev() const; + std::optional getTreeHash() const; std::optional getRevCount() const; std::optional getLastModified() const; }; @@ -148,14 +150,14 @@ struct InputScheme virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); - virtual std::pair fetch(ref store, const Input & input) = 0; + virtual std::pair fetch(ref store, const Input & input) = 0; }; void registerInputScheme(std::shared_ptr && fetcher); struct DownloadFileResult { - StorePath storePath; + StorePathDescriptor storePath; std::string etag; std::string effectiveUrl; std::optional immutableUrl; @@ -182,4 +184,7 @@ DownloadTarballResult downloadTarball( bool locked, const Headers & headers = {}); +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name); + } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f8d89ab2fcd..11098eacfd6 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" +#include "git.hh" #include "url-parts.hh" #include "pathlocks.hh" #include "util.hh" @@ -200,7 +201,7 @@ WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) return WorkdirInfo { .clean = clean, .hasHead = hasHead }; } -std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) +std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo, FileIngestionMethod ingestionMethod) { const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); auto gitDir = ".git"; @@ -235,7 +236,10 @@ std::pair fetchFromWorkdir(ref store, Input & input, co return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, ingestionMethod, htSHA256, filter); + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + auto storePathDesc = store->queryPathInfo(storePath)->fullStorePathDescriptorOpt().value(); // FIXME: maybe we should use the timestamp of the last // modified dirty file? @@ -250,7 +254,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty"); } - return {std::move(storePath), input}; + return {std::move(storePathDesc), input}; } } // end namespace @@ -272,7 +276,7 @@ struct GitInputScheme : InputScheme attrs.emplace("type", "git"); for (auto & [name, value] : url.query) { - if (name == "rev" || name == "ref") + if (name == "rev" || name == "ref" || name == "treeHash" || name == "gitIngestion") attrs.emplace(name, value); else if (name == "shallow" || name == "submodules" || name == "allRefs") attrs.emplace(name, Explicit { value == "1" }); @@ -290,7 +294,7 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "gitIngestion" && name != "treeHash" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") throw Error("unsupported Git input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); @@ -313,6 +317,9 @@ struct GitInputScheme : InputScheme auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme != "git") url.scheme = "git+" + url.scheme; if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto treeHash = input.getTreeHash()) url.query.insert_or_assign("treeHash", treeHash->gitRev()); + if (maybeGetBoolAttr(input.attrs, "gitIngestion").value_or((bool) input.getTreeHash())) + url.query.insert_or_assign("gitIngestion", "1"); if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) url.query.insert_or_assign("shallow", "1"); @@ -323,11 +330,18 @@ struct GitInputScheme : InputScheme { bool maybeDirty = !input.getRef(); bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - return + bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + /* FIXME just requiring tree hash is necessary for substitutions to + work for now, but breaks eval purity. Need a better solution before + upstreaming. */ + return (input.getTreeHash() && !submodules) || ( maybeGetIntAttr(input.attrs, "lastModified") - && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); + && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")) + && input.getNarHash()); } + /* FIXME no overriding the tree hash / flake registry support for tree + hashes, for now. */ Input applyOverrides( const Input & input, std::optional ref, @@ -364,7 +378,7 @@ struct GitInputScheme : InputScheme std::optional getSourcePath(const Input & input) override { auto url = parseURL(getStrAttr(input.attrs, "url")); - if (url.scheme == "file" && !input.getRef() && !input.getRev()) + if (url.scheme == "file" && !input.getRef() && !input.getRev() && !input.getTreeHash()) return url.path; return {}; } @@ -396,7 +410,7 @@ struct GitInputScheme : InputScheme return {isLocal, isLocal ? url.path : url.base}; } - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & _input) override { Input input(_input); auto gitDir = ".git"; @@ -412,6 +426,11 @@ struct GitInputScheme : InputScheme if (submodules) cacheType += "-submodules"; if (allRefs) cacheType += "-all-refs"; + auto ingestionMethod = + maybeGetBoolAttr(input.attrs, "gitIngestion").value_or((bool) input.getTreeHash()) + ? FileIngestionMethod::Git + : FileIngestionMethod::Recursive; + auto checkHashType = [&](const std::optional & hash) { if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) @@ -420,24 +439,36 @@ struct GitInputScheme : InputScheme auto getLockedAttrs = [&]() { - checkHashType(input.getRev()); - - return Attrs({ + Attrs attrs({ {"type", cacheType}, {"name", name}, - {"rev", input.getRev()->gitRev()}, }); + if (auto optH = input.getTreeHash()) { + auto h = *std::move(optH); + checkHashType(h); + attrs.insert_or_assign("treeHash", h.gitRev()); + } + if (auto optH = input.getRev()) { + auto h = *std::move(optH); + checkHashType(h); + attrs.insert_or_assign("rev", h.gitRev()); + } + if (maybeGetBoolAttr(input.attrs, "gitIngestion").value_or((bool) input.getTreeHash())) + attrs.insert_or_assign("gitIngestion", true); + return attrs; }; - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair + auto makeResult = [&](const Attrs & infoAttrs, StorePathDescriptor && storePathDesc) + -> std::pair { - assert(input.getRev()); + assert(input.getRev() || input.getTreeHash()); + /* If was originally set, that original value must be preserved. */ assert(!_input.getRev() || _input.getRev() == input.getRev()); + assert(!_input.getTreeHash() || _input.getTreeHash() == input.getTreeHash()); if (!shallow) input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); - return {std::move(storePath), input}; + return {std::move(storePathDesc), input}; }; if (input.getRev()) { @@ -450,10 +481,10 @@ struct GitInputScheme : InputScheme /* If this is a local directory and no ref or revision is given, allow fetching directly from a dirty workdir. */ - if (!input.getRef() && !input.getRev() && isLocal) { + if (!input.getRef() && !input.getRev() && !input.getTreeHash() && isLocal) { auto workdirInfo = getWorkdirInfo(input, actualUrl); if (!workdirInfo.clean) { - return fetchFromWorkdir(store, input, actualUrl, workdirInfo); + return fetchFromWorkdir(store, input, actualUrl, workdirInfo, ingestionMethod); } } @@ -462,6 +493,8 @@ struct GitInputScheme : InputScheme {"name", name}, {"url", actualUrl}, }); + if (ingestionMethod == FileIngestionMethod::Git) + unlockedAttrs.insert_or_assign("gitIngestion", true); Path repoDir; @@ -476,9 +509,14 @@ struct GitInputScheme : InputScheme unlockedAttrs.insert_or_assign("ref", *head); } - if (!input.getRev()) - input.attrs.insert_or_assign("rev", - Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev()); + if (!input.getRev() && !input.getTreeHash()) { + auto getHash = [&](std::string rev) { + return Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", rev })), htSHA1).gitRev(); + }; + input.attrs.insert_or_assign("rev", getHash(*input.getRef())); + if (experimentalFeatureSettings.isEnabled(Xp::GitHashing)) + input.attrs.insert_or_assign("treeHash", getHash(*input.getRef() + ":")); + } repoDir = actualUrl; } else { @@ -498,11 +536,33 @@ struct GitInputScheme : InputScheme } if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); - if (!input.getRev() || input.getRev() == rev2) { - input.attrs.insert_or_assign("rev", rev2.gitRev()); - return makeResult(res->first, std::move(res->second)); + bool found = false; + + if (std::optional revS = maybeGetStrAttr(res->first, "rev")) { + auto rev2 = Hash::parseAny(*revS, htSHA1); + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); + found = true; + } + } + + if (experimentalFeatureSettings.isEnabled(Xp::GitHashing)) { + if (std::optional treeHashS = maybeGetStrAttr(res->first, "treeHash")) { + auto treeHash2 = Hash::parseNonSRIUnprefixed(*treeHashS, htSHA1); + if (!input.getTreeHash() || input.getTreeHash() == treeHash2) { + input.attrs.insert_or_assign("treeHash", treeHash2.gitRev()); + found = true; + } + } } + + bool correctIngestion = + maybeGetBoolAttr(input.attrs, "gitIngestion").value_or(false) + ? ingestionMethod == FileIngestionMethod::Git + : ingestionMethod == FileIngestionMethod::Recursive; + + if (correctIngestion && found) + return makeResult(res->first, std::move(res->second)); } Path cacheDir = getCachePath(actualUrl); @@ -524,11 +584,12 @@ struct GitInputScheme : InputScheme bool doFetch; time_t now = time(0); - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (input.getRev()) { + /* If a rev or treeHash is specified, we need to fetch if + it's not in the repo. */ + if (input.getRev() || input.getTreeHash()) { try { - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() }); + auto fetchHash = input.getTreeHash() ? input.getTreeHash() : input.getRev(); + runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", fetchHash->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -575,12 +636,23 @@ struct GitInputScheme : InputScheme warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl); } - if (!input.getRev()) - input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + if (!input.getRev() && !input.getTreeHash()) { + auto rev = Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev(); + input.attrs.insert_or_assign("rev", rev); + if (experimentalFeatureSettings.isEnabled(Xp::GitHashing)) + input.attrs.insert_or_assign("treeHash", + Hash::parseAny(chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", rev + ":" })), htSHA1).gitRev()); + } // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } + if (input.getTreeHash()) { + auto type = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-t", input.getTreeHash()->gitRev() })); + if (type != "tree") + throw Error("Need a tree object, found '%s' object in %s", type, input.getTreeHash()->gitRev()); + } + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true"; if (isShallow && !shallow) @@ -588,7 +660,10 @@ struct GitInputScheme : InputScheme // FIXME: check whether rev is an ancestor of ref. - printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); + if (auto rev = input.getRev()) + printTalkative("using revision %s of repo '%s'", rev->gitRev(), actualUrl); + if (auto treeHash = input.getTreeHash()) + printTalkative("using tree %s of repo '%s'", treeHash->gitRev(), actualUrl); /* Now that we know the ref, check again whether we have it in the store. */ @@ -599,10 +674,20 @@ struct GitInputScheme : InputScheme AutoDelete delTmpDir(tmpDir, true); PathFilter filter = defaultPathFilter; + auto [fetchHash, fetchHashType] = input.getTreeHash() + ? (std::pair { input.getTreeHash().value(), true }) + : (std::pair { input.getRev().value(), false }); + auto result = runProgram(RunOptions { .program = "git", - .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() }, - .mergeStderrToStdout = true + .args = { + "-C", repoDir, + "--git-dir", gitDir, + "cat-file", + fetchHashType ? "tree" : "commit", + fetchHash.gitRev(), + }, + .mergeStderrToStdout = true, }); if (WEXITSTATUS(result.first) == 128 && result.second.find("bad file") != std::string::npos) @@ -612,13 +697,18 @@ struct GitInputScheme : InputScheme "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", - input.getRev()->gitRev(), + fetchHash.gitRev(), *input.getRef(), actualUrl ); } if (submodules) { + if (input.getTreeHash()) + throw Error("Cannot fetch specific tree hashes if there are submodules"); + if (ingestionMethod == FileIngestionMethod::Git) + warn("Nix's computed git tree hash will be different when submodules are converted to regular directories"); + Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); @@ -633,7 +723,7 @@ struct GitInputScheme : InputScheme "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true); } - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getTreeHash() ? input.getTreeHash()->gitRev() : input.getRev()->gitRev() }); /* Ensure that we use the correct origin for fetching submodules. This matters for submodules with relative @@ -662,12 +752,13 @@ struct GitInputScheme : InputScheme filter = isNotDotGitDirectory; } else { + auto & fetchHash_ = fetchHash; // Work-around clang restriction. // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { runProgram2({ .program = "git", - .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() }, + .args = { "-C", repoDir, "--git-dir", gitDir, "archive", fetchHash_.gitRev() }, .standardOut = &sink }); }); @@ -675,35 +766,57 @@ struct GitInputScheme : InputScheme unpackTarfile(*source, tmpDir); } - auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(name, tmpDir, ingestionMethod, ingestionMethod == FileIngestionMethod::Git ? htSHA1 : htSHA256, filter); + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + auto storePathDesc = store->queryPathInfo(storePath)->fullStorePathDescriptorOpt().value(); + + // verify treeHash is what we actually obtained in the nix store + if (auto treeHash = input.getTreeHash()) { + auto path = store->toRealPath(store->printStorePath(storePath)); + auto gotHash = dumpGitHash(htSHA1, path); + if (gotHash != input.getTreeHash()) + throw Error("Git hash mismatch in input '%s' (%s), expected '%s', got '%s'", + input.to_string(), path, treeHash->gitRev(), gotHash.gitRev()); + } - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); + Attrs infoAttrs({}); - Attrs infoAttrs({ - {"rev", input.getRev()->gitRev()}, - {"lastModified", lastModified}, - }); + if (auto rev = input.getRev()) { + infoAttrs.insert_or_assign("rev", rev->gitRev()); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", rev->gitRev() })); + infoAttrs.insert_or_assign("lastModified", lastModified); + } else + infoAttrs.insert_or_assign("lastModified", (uint64_t) 0); - if (!shallow) - infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() }))); + if (experimentalFeatureSettings.isEnabled(Xp::GitHashing)) + if (auto treeHash = input.getTreeHash()) + infoAttrs.insert_or_assign("treeHash", treeHash->gitRev()); + + if (!shallow) { + if (auto rev = input.getRev()) + infoAttrs.insert_or_assign("revCount", + std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", rev->gitRev() }))); + else + infoAttrs.insert_or_assign("revCount", (uint64_t) 0); + } - if (!_input.getRev()) + if (!_input.getRev() && !_input.getTreeHash()) getCache()->add( store, unlockedAttrs, infoAttrs, - storePath, + storePathDesc, false); getCache()->add( store, getLockedAttrs(), infoAttrs, - storePath, + storePathDesc, true); - return makeResult(infoAttrs, std::move(storePath)); + return makeResult(infoAttrs, std::move(storePathDesc)); } }; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 291f457f0d3..5a55933e40e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -134,7 +134,7 @@ struct GitArchiveInputScheme : InputScheme bool hasAllInfo(const Input & input) const override { - return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); + return input.getNarHash() && input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); } Input applyOverrides( @@ -183,7 +183,7 @@ struct GitArchiveInputScheme : InputScheme virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & _input) override { Input input(_input); @@ -267,9 +267,8 @@ struct GitHubInputScheme : GitArchiveInputScheme Headers headers = makeHeadersWithAuthTokens(host); auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); + readFile(store->toRealPath(store->makeFixedOutputPathFromCA( + downloadFile(store, url, "source", false, headers).storePath)))); auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; @@ -338,10 +337,9 @@ struct GitLabInputScheme : GitArchiveInputScheme Headers headers = makeHeadersWithAuthTokens(host); - auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); + auto json = nlohmann::json::parse(readFile( + store->toRealPath(store->makeFixedOutputPathFromCA( + downloadFile(store, url, "source", false, headers).storePath)))); auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; @@ -403,8 +401,8 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string refUri; if (ref == "HEAD") { - auto file = store->toRealPath( - downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath); + auto file = store->toRealPath(store->makeFixedOutputPathFromCA( + downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath)); std::ifstream is(file); std::string line; getline(is, line); @@ -419,8 +417,8 @@ struct SourceHutInputScheme : GitArchiveInputScheme } std::regex refRegex(refUri); - auto file = store->toRealPath( - downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); + auto file = store->toRealPath(store->makeFixedOutputPathFromCA( + downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath)); std::ifstream is(file); std::string line; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 4874a43ff97..2e56520296c 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -94,7 +94,7 @@ struct IndirectInputScheme : InputScheme return input; } - std::pair fetch(ref store, const Input & input) override + std::pair fetch(ref store, const Input & input) override { throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 51fd1ed428b..806e2ed8c28 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -102,7 +102,8 @@ struct MercurialInputScheme : InputScheme { // FIXME: ugly, need to distinguish between dirty and clean // default trees. - return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); + return input.getNarHash() + && (input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount")); } Input applyOverrides( @@ -145,7 +146,7 @@ struct MercurialInputScheme : InputScheme return {isLocal, isLocal ? url.path : url.base}; } - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & _input) override { Input input(_input); @@ -196,8 +197,11 @@ struct MercurialInputScheme : InputScheme }; auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + auto storePathDesc = store->queryPathInfo(storePath)->fullStorePathDescriptorOpt().value(); - return {std::move(storePath), input}; + return {std::move(storePathDesc), input}; } } @@ -221,8 +225,8 @@ struct MercurialInputScheme : InputScheme }); }; - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair + auto makeResult = [&](const Attrs & infoAttrs, StorePathDescriptor && storePath) + -> std::pair { assert(input.getRev()); assert(!_input.getRev() || _input.getRev() == input.getRev()); @@ -301,6 +305,9 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir + "/.hg_archival.txt"); auto storePath = store->addToStore(name, tmpDir); + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + auto storePathDesc = store->queryPathInfo(storePath)->fullStorePathDescriptorOpt().value(); Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, @@ -312,17 +319,17 @@ struct MercurialInputScheme : InputScheme store, unlockedAttrs, infoAttrs, - storePath, + storePathDesc, false); getCache()->add( store, getLockedAttrs(), infoAttrs, - storePath, + storePathDesc, true); - return makeResult(infoAttrs, std::move(storePath)); + return makeResult(infoAttrs, std::move(storePathDesc)); } }; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 01f1be97822..caeb6c15518 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -68,7 +68,7 @@ struct PathInputScheme : InputScheme bool hasAllInfo(const Input & input) const override { - return true; + return (bool) input.getNarHash(); } std::optional getSourcePath(const Input & input) override @@ -81,7 +81,7 @@ struct PathInputScheme : InputScheme // nothing to do } - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & _input) override { Input input(_input); std::string absPath; @@ -123,7 +123,11 @@ struct PathInputScheme : InputScheme } input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - return {std::move(*storePath), input}; + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + auto storePathDesc = store->queryPathInfo(*storePath)->fullStorePathDescriptorOpt().value(); + + return {std::move(storePathDesc), input}; } }; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 43c03beec17..74df2cd3a6e 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -158,7 +158,8 @@ static std::shared_ptr getGlobalRegistry(ref store) } if (!hasPrefix(path, "/")) { - auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + auto storePathDesc = downloadFile(store, path, "flake-registry.json", false).storePath; + auto storePath = store->makeFixedOutputPathFromCA(storePathDesc); if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json"); path = store->toRealPath(storePath); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e3a41e75e44..8d33290a891 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -64,7 +64,7 @@ DownloadFileResult downloadFile( if (res.immutableUrl) infoAttrs.emplace("immutableUrl", *res.immutableUrl); - std::optional storePath; + std::optional storePath; if (res.cached) { assert(cached); @@ -73,20 +73,22 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(res.data, sink); auto hash = hashString(htSHA256, res.data); - ValidPathInfo info { - *store, - name, - FixedOutputInfo { + storePath = { + .name = name, + .info = FixedOutputInfo { .method = FileIngestionMethod::Flat, .hash = hash, .references = {}, }, + }; + ValidPathInfo info { + *store, + StorePathDescriptor { *storePath }, hashString(htSHA256, sink.s), }; info.narSize = sink.s.size(); auto source = StringSource { sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); - storePath = std::move(info.path); } getCache()->add( @@ -133,14 +135,17 @@ DownloadTarballResult downloadTarball( if (cached && !cached->expired) return { - .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, + .tree = Tree { + .actualPath = store->toRealPath(store->makeFixedOutputPathFromCA(cached->storePath)), + .storePath = std::move(cached->storePath), + }, .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; auto res = downloadFile(store, url, name, locked, headers); - std::optional unpackedStorePath; + std::optional unpackedStorePath; time_t lastModified; if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { @@ -149,13 +154,18 @@ DownloadTarballResult downloadTarball( } else { Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(res.storePath), tmpDir); + unpackTarfile( + store->toRealPath(store->makeFixedOutputPathFromCA(res.storePath)), + tmpDir); auto members = readDirectory(tmpDir); if (members.size() != 1) throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + auto temp = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + unpackedStorePath = store->queryPathInfo(temp)->fullStorePathDescriptorOpt().value(); } Attrs infoAttrs({ @@ -174,7 +184,10 @@ DownloadTarballResult downloadTarball( locked); return { - .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, + .tree = Tree { + .actualPath = store->toRealPath(store->makeFixedOutputPathFromCA(*unpackedStorePath)), + .storePath = std::move(*unpackedStorePath) + }, .lastModified = lastModified, .immutableUrl = res.immutableUrl, }; @@ -274,7 +287,7 @@ struct FileInputScheme : CurlInputScheme : (!requireTree && !hasTarballExtension(url.path))); } - std::pair fetch(ref store, const Input & input) override + std::pair fetch(ref store, const Input & input) override { auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); return {std::move(file.storePath), input}; @@ -295,7 +308,7 @@ struct TarballInputScheme : CurlInputScheme : (requireTree || hasTarballExtension(url.path))); } - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & _input) override { Input input(_input); auto url = getStrAttr(input.attrs, "url"); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b4fea693f5b..95c926b31d5 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,11 +180,11 @@ ref BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. */ - for (auto & ref : info.references) + reads, but typically they'll already be cached. Note that + self-references are always valid. */ + for (auto & ref : info.references.others) try { - if (ref != info.path) - queryPathInfo(ref); + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); @@ -307,14 +307,16 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - name, - FixedOutputInfo { - .method = method, - .hash = nar.first, - .references = { - .others = references, + { + .name = std::string { name }, + .info = FixedOutputInfo { + .method = method, + .hash = nar.first, + .references = { + .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus - .self = false, + .self = false, + }, }, }, nar.first, @@ -324,12 +326,12 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n })->path; } -bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) +bool BinaryCacheStore::isValidPathUncached(StorePathOrDesc storePath) { // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even // though they shouldn't. Not easily fixed. - return fileExists(narInfoFileFor(storePath)); + return fileExists(narInfoFileFor(bakeCaIfNeeded(storePath))); } std::optional BinaryCacheStore::queryPathFromHashPart(const std::string & hashPart) @@ -343,7 +345,7 @@ std::optional BinaryCacheStore::queryPathFromHashPart(const std::stri } } -void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) +void BinaryCacheStore::narFromPath(StorePathOrDesc storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast(); @@ -365,16 +367,17 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) stats.narReadBytes += narSize.length; } -void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, +void BinaryCacheStore::queryPathInfoUncached(StorePathOrDesc storePath, Callback> callback) noexcept { auto uri = getUri(); - auto storePathS = printStorePath(storePath); + auto actualStorePath = bakeCaIfNeeded(storePath); + auto storePathS = printStorePath(actualStorePath); auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); PushActivity pact(act->id); - auto narInfoFile = narInfoFileFor(storePath); + auto narInfoFile = narInfoFileFor(actualStorePath); auto callbackPtr = std::make_shared(std::move(callback)); @@ -411,10 +414,15 @@ StorePath BinaryCacheStore::addToStore( implementation of this method in terms of addToStoreFromDump. */ HashSink sink { hashAlgo }; - if (method == FileIngestionMethod::Recursive) { + switch (method) { + case FileIngestionMethod::Recursive: dumpPath(srcPath, sink, filter); - } else { + break; + case FileIngestionMethod::Flat: readFile(srcPath, sink); + break; + case FileIngestionMethod::Git: + throw Error("cannot add to binary cache store using the git file ingestion method"); } auto h = sink.finish().first; @@ -424,14 +432,16 @@ StorePath BinaryCacheStore::addToStore( return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - name, - FixedOutputInfo { - .method = method, - .hash = h, - .references = { - .others = references, - // caller is not capable of creating a self-reference, because this is content-addressed without modulus - .self = false, + { + .name = std::string { name }, + .info = FixedOutputInfo { + .method = method, + .hash = h, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, }, }, nar.first, @@ -459,10 +469,12 @@ StorePath BinaryCacheStore::addTextToStore( return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - std::string { name }, - TextInfo { - .hash = textHash, - .references = references, + { + .name = std::string { name }, + .info = TextInfo { + .hash = textHash, + .references = references, + }, }, nar.first, }; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 49f271d248c..2bc2a4c6f20 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -113,9 +113,9 @@ private: public: - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept override; std::optional queryPathFromHashPart(const std::string & hashPart) override; @@ -146,7 +146,7 @@ public: void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override; - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(StorePathOrDesc path, Sink & sink) override; ref getFSAccessor() override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bec0bc538ab..b4f02aa42d6 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -258,11 +258,12 @@ void DerivationGoal::haveDerivation() ) ); else { - auto * cap = getDerivationCA(*drv); + auto optCA = getDerivationCA(*drv); + auto storePathOrCA = + optCA ? StorePathOrDesc { *optCA } : status.known->path; addWaitee(upcast_goal(worker.makePathSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - cap ? std::optional { *cap } : std::nullopt))); + storePathOrCA, + buildMode == bmRepair ? Repair : NoRepair))); } } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index f0f0e551935..8b35474f209 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -92,7 +92,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat } -void Store::ensurePath(const StorePath & path) +void Store::ensurePath(StorePathOrDesc path) { /* If the path is already valid, we're done. */ if (isValidPath(path)) return; @@ -107,8 +107,10 @@ void Store::ensurePath(const StorePath & path) if (goal->ex) { goal->ex->status = worker.failingExitStatus(); throw std::move(*goal->ex); - } else - throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + } else { + auto p = this->bakeCaIfNeeded(path); + throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(p)); + } } } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 64b55ca6ac2..be55697fc9e 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -8,6 +8,7 @@ #include "finally.hh" #include "util.hh" #include "archive.hh" +#include "git.hh" #include "compression.hh" #include "daemon.hh" #include "topo-sort.hh" @@ -1244,10 +1245,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In return paths; } - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept override { - if (goal.isAllowed(path)) { + if (goal.isAllowed(bakeCaIfNeeded(path))) { try { /* Censor impure information. */ auto info = std::make_shared(*next->queryPathInfo(path)); @@ -1319,15 +1320,17 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In return path; } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override { + auto path = bakeCaIfNeeded(pathOrDesc); if (!goal.isAllowed(path)) throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); + LocalFSStore::narFromPath(pathOrDesc, sink); } - void ensurePath(const StorePath & path) override + void ensurePath(StorePathOrDesc pathOrDesc) override { + auto path = bakeCaIfNeeded(pathOrDesc); if (!goal.isAllowed(path)) throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); /* Nothing to be done; 'path' must already be valid. */ @@ -2480,23 +2483,34 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; + Hash got { outputHash.hashType }; // Dummy value std::visit(overloaded { [&](const TextIngestionMethod &) { + HashModuloSink caSink { outputHash.hashType, oldHashPart }; readFile(actualPath, caSink); + got = caSink.finish().first; }, [&](const FileIngestionMethod & m2) { switch (m2) { - case FileIngestionMethod::Recursive: + case FileIngestionMethod::Recursive: { + HashModuloSink caSink { outputHash.hashType, oldHashPart }; dumpPath(actualPath, caSink); + got = caSink.finish().first; break; - case FileIngestionMethod::Flat: + } + case FileIngestionMethod::Flat: { + HashModuloSink caSink { outputHash.hashType, oldHashPart }; readFile(actualPath, caSink); + got = caSink.finish().first; + break; + } + case FileIngestionMethod::Git: { + got = dumpGitHash(outputHash.hashType, (Path) tmpDir + "/tmp"); break; } + } }, }, outputHash.method.raw); - auto got = caSink.finish().first; auto optCA = ContentAddressWithReferences::fromPartsOpt( outputHash.method, @@ -2510,8 +2524,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() } ValidPathInfo newInfo0 { worker.store, - outputPathName(drv->name, outputName), - *std::move(optCA), + { + .name = outputPathName(drv->name, outputName), + .info = *std::move(optCA), + }, Hash::dummy, }; if (*scratchPath != newInfo0.path) { @@ -2548,10 +2564,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = std::move(refs.others); - if (refs.self) - newInfo0.references.insert(newInfo0.path); + newInfo0.references = rewriteRefs(); return newInfo0; }, @@ -2807,12 +2820,12 @@ void LocalDerivationGoal::checkOutputs(const std::mapsecond.narSize; - for (auto & ref : i->second.references) + for (auto & ref : i->second.referencesPossiblyToSelf()) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) pathsLeft.push(ref); } } @@ -2852,7 +2865,7 @@ void LocalDerivationGoal::checkOutputs(const std::map ca) +PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker, DerivedPath::Opaque { storePath }) , storePath(storePath) , repair(repair) @@ -95,9 +95,7 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA( - std::string { storePath.name() }, - ContentAddressWithReferences::withoutRefs(*ca)); + auto subPath = sub->makeFixedOutputPathFromCA(*ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { @@ -107,7 +105,8 @@ void PathSubstitutionGoal::tryNext() try { // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); + auto p = ca ? StorePathOrDesc { std::cref(*ca) } : std::cref(storePath); + info = sub->queryPathInfo(p); } catch (InvalidPath &) { tryNext(); return; @@ -164,9 +163,8 @@ void PathSubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + for (auto & i : info->references.others) + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -187,9 +185,8 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + for (auto & i : info->references.others) + assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); @@ -223,8 +220,10 @@ void PathSubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); + auto p = ca ? StorePathOrDesc { std::cref(*ca) } : std::cref(storePath); + copyStorePath(*sub, worker.store, - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + p, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 1d389d328ff..efb8c6cf8d7 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -13,15 +13,11 @@ struct PathSubstitutionGoal : public Goal { /** * The store path that should be realised through a substitute. + * + * @todo should be `OwnedStorePathOrDesc`. */ StorePath storePath; - /** - * The path the substituter refers to the path as. This will be - * different when the stores have different names. - */ - std::optional subPath; - /** * The remaining substituters. */ @@ -73,8 +69,10 @@ struct PathSubstitutionGoal : public Goal /** * Content address for recomputing store path + * + * @todo delete once `storePath` is a `std::variant`. */ - std::optional ca; + std::optional ca; void done( ExitCode result, @@ -82,7 +80,7 @@ struct PathSubstitutionGoal : public Goal std::optional errorMsg = {}); public: - PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~PathSubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index f65f63b9947..07165c9dfee 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -100,25 +100,30 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -std::shared_ptr Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makePathSubstitutionGoal(StorePathOrDesc path, RepairFlag repair) { - std::weak_ptr & goal_weak = substitutionGoals[path]; - auto goal = goal_weak.lock(); // FIXME + auto p = store.bakeCaIfNeeded(path); + std::weak_ptr & goal_weak = substitutionGoals[p]; + std::shared_ptr goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared(path, *this, repair, ca); + auto optCA = std::get_if<1>(&path); + goal = std::make_shared( + p, + *this, + repair, + optCA ? std::optional { *optCA } : std::nullopt); goal_weak = goal; wakeUp(goal); } return goal; } - -std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair) { std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; auto goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared(id, *this, repair, ca); + goal = std::make_shared(id, *this, repair); goal_weak = goal; wakeUp(goal); } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index a778e311c18..3768b4ebcb6 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -225,8 +225,8 @@ public: /** * @ref SubstitutionGoal "substitution goal" */ - std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makePathSubstitutionGoal(StorePathOrDesc storePath, RepairFlag repair = NoRepair); + std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair); /** * Make a goal corresponding to the `DerivedPath`. diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index e290a8d387e..37598e0348b 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -11,6 +11,8 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; + case FileIngestionMethod::Git: + return "git:"; default: throw Error("impossible, caught both cases"); } @@ -32,6 +34,8 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) ContentAddressMethod method = FileIngestionMethod::Flat; if (splitPrefix(m, "r:")) method = FileIngestionMethod::Recursive; + if (splitPrefix(m, "git:")) + method = FileIngestionMethod::Git; else if (splitPrefix(m, "text:")) method = TextIngestionMethod {}; return method; @@ -63,6 +67,21 @@ std::string ContentAddress::render() const + this->hash.to_string(Base32, true); } +static HashType parseHashType_(std::string_view & rest) { + auto hashTypeRaw = splitPrefixTo(rest, ':'); + if (!hashTypeRaw) + throw UsageError("hash must be in form \":\", but found: %s", rest); + return parseHashType(*hashTypeRaw); +}; + +static FileIngestionMethod parseFileIngestionMethod_(std::string_view & rest) { + if (splitPrefix(rest, "r:")) + return FileIngestionMethod::Recursive; + else if (splitPrefix(rest, "git:")) + return FileIngestionMethod::Git; + return FileIngestionMethod::Flat; +} + /** * Parses content address strings up to the hash. */ @@ -74,22 +93,14 @@ static std::pair parseContentAddressMethodPrefix { auto optPrefix = splitPrefixTo(rest, ':'); if (!optPrefix) - throw UsageError("not a content address because it is not in the form ':': %s", wholeInput); + throw UsageError("not a path-info content address because it is not in the form \":\": %s", wholeInput); prefix = *optPrefix; } - auto parseHashType_ = [&](){ - auto hashTypeRaw = splitPrefixTo(rest, ':'); - if (!hashTypeRaw) - throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); - HashType hashType = parseHashType(*hashTypeRaw); - return std::move(hashType); - }; - // Switch on prefix if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. - HashType hashType = parseHashType_(); + HashType hashType = parseHashType_(rest); return { TextIngestionMethod {}, std::move(hashType), @@ -99,13 +110,15 @@ static std::pair parseContentAddressMethodPrefix auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; - HashType hashType = parseHashType_(); + if (splitPrefix(rest, "git:")) + method = FileIngestionMethod::Git; + HashType hashType = parseHashType_(rest); return { std::move(method), std::move(hashType), }; } else - throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); + throw UsageError("path-info content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix); } ContentAddress ContentAddress::parse(std::string_view rawCa) @@ -140,20 +153,119 @@ std::string renderContentAddress(std::optional ca) return ca ? ca->render() : ""; } -std::string ContentAddress::printMethodAlgo() const + +// FIXME Deduplicate with store-api.cc path computation +std::string renderStorePathDescriptor(StorePathDescriptor ca) { - return method.renderPrefix() - + printHashType(hash.type); + std::string result { ca.name }; + result += ":"; + + auto dumpRefs = [&](auto references, bool hasSelfReference) { + result += "refs:"; + result += std::to_string(references.size()); + for (auto & i : references) { + result += ":"; + result += i.to_string(); + } + if (hasSelfReference) result += ":self"; + result += ":"; + }; + + std::visit(overloaded { + [&](const TextInfo & th) { + result += "text:"; + dumpRefs(th.references, false); + result += th.hash.to_string(Base32, true); + }, + [&](const FixedOutputInfo & fsh) { + result += "fixed:"; + dumpRefs(fsh.references.others, fsh.references.self); + result += makeFileIngestionPrefix(fsh.method); + result += fsh.hash.to_string(Base32, true); + }, + }, ca.info.raw); + + return result; } -bool StoreReferences::empty() const + +StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa) { - return !self && others.empty(); + warn("%s", rawCa); + auto rest = rawCa; + + std::string_view name; + std::string_view tag; + { + auto optName = splitPrefixTo(rest, ':'); + auto optTag = splitPrefixTo(rest, ':'); + if (!(optTag && optName)) + throw UsageError("not a content address because it is not in the form \"::\": %s", rawCa); + tag = *optTag; + name = *optName; + } + + auto parseRefs = [&]() -> StoreReferences { + if (!splitPrefix(rest, "refs:")) + throw Error("Invalid CA \"%s\", \"%s\" should begin with \"refs:\"", rawCa, rest); + StoreReferences ret; + size_t numReferences = 0; + { + auto countRaw = splitPrefixTo(rest, ':'); + if (!countRaw) + throw UsageError("Invalid count"); + numReferences = std::stoi(std::string { *countRaw }); + } + for (size_t i = 0; i < numReferences; i++) { + auto s = splitPrefixTo(rest, ':'); + if (!s) + throw UsageError("Missing reference no. %d", i); + ret.others.insert(StorePath(*s)); + } + if (splitPrefix(rest, "self:")) + ret.self = true; + return ret; + }; + + // Dummy value + ContentAddressWithReferences info = TextInfo { Hash(htSHA256), {} }; + + // Switch on tag + if (tag == "text") { + auto refs = parseRefs(); + if (refs.self) + throw UsageError("Text content addresses cannot have self references"); + auto hashType = parseHashType_(rest); + if (hashType != htSHA256) + throw Error("Text content address hash should use %s, but instead uses %s", + printHashType(htSHA256), printHashType(hashType)); + info = TextInfo { + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), + .references = refs.others, + }; + } else if (tag == "fixed") { + auto refs = parseRefs(); + auto method = parseFileIngestionMethod_(rest); + auto hashType = parseHashType_(rest); + info = FixedOutputInfo { + .method = method, + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), + .references = refs, + }; + } else + throw UsageError("content address tag \"%s\" is unrecognized. Recogonized tages are \"text\" or \"fixed\"", tag); + + return StorePathDescriptor { + .name = std::string { name }, + .info = info, + }; } -size_t StoreReferences::size() const + +std::string ContentAddress::printMethodAlgo() const { - return (self ? 1 : 0) + others.size(); + return method.renderPrefix() + + printHashType(hash.type); } ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index c4d619bdc63..63865e3da15 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -5,6 +5,7 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" +#include "reference-set.hh" #include "variant-wrapper.hh" namespace nix { @@ -39,12 +40,26 @@ enum struct FileIngestionMethod : uint8_t { /** * Flat-file hashing. Directly ingest the contents of a single file */ - Flat = false, + Flat, + /** * Recursive (or NAR) hashing. Serializes the file-system object in Nix * Archive format and ingest that */ - Recursive = true + Recursive, + + /** + * Git hashing. In particular files are hashed as git "blobs", and + * directories are hashed as git "trees". + * + * @note Git's data model is slightly different, in that a plain + * fail doesn't have an executable bit, directory entries do + * instead. We decide treat a bare file as non-executable by fiat, + * as we do with `FileIngestionMethod::Flat` which also lacks this + * information. Thus, Git can encode some but all of Nix's "File + * System Objects", and this sort of hashing is likewise partial. + */ + Git, }; /** @@ -163,38 +178,7 @@ std::string renderContentAddress(std::optional ca); * See the schema for store paths in store-api.cc */ -/** - * A set of references to other store objects. - * - * References to other store objects are tracked with store paths, self - * references however are tracked with a boolean. - */ -struct StoreReferences -{ - /** - * References to other store objects - */ - StorePathSet others; - - /** - * Reference to this store object - */ - bool self = false; - - /** - * @return true iff no references, i.e. others is empty and self is - * false. - */ - bool empty() const; - - /** - * Returns the numbers of references, i.e. the size of others + 1 - * iff self is true. - */ - size_t size() const; - - GENERATE_CMP(StoreReferences, me->self, me->others); -}; +using StoreReferences = References; // This matches the additional info that we need for makeTextPath struct TextInfo @@ -277,4 +261,15 @@ struct ContentAddressWithReferences Hash getHash() const; }; +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + GENERATE_CMP(StorePathDescriptor, me->name, me->info); +}; + +std::string renderStorePathDescriptor(StorePathDescriptor ca); + +StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa); + } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8cbf6f044fa..fe284a2cf2c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -13,6 +13,7 @@ #include "archive.hh" #include "derivations.hh" #include "args.hh" +#include "git.hh" namespace nix::daemon { @@ -346,7 +347,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); StorePathSet paths; if (op == WorkerProto::Op::QueryReferences) - for (auto & i : store->queryPathInfo(path)->references) + for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) paths.insert(i); else if (op == WorkerProto::Op::QueryReferrers) store->queryReferrers(path, paths); @@ -462,13 +463,17 @@ static void performOp(TunnelLogger * logger, ref store, TeeSource savedNARSource(from, saved); ParseSink sink; /* null sink; just parse the NAR */ parseDump(sink, savedNARSource); - } else { + } else if (method == FileIngestionMethod::Flat) { /* Incrementally parse the NAR file, stripping the metadata, and streaming the sole file we expect into `saved`. */ RetrieveRegularNARSink savedRegular { saved }; parseDump(savedRegular, from); if (!savedRegular.regular) throw Error("regular file expected"); + } else { + /* Should have validated above that no other file ingestion + method was used. */ + assert(false); } }); logger->startWork(); @@ -776,7 +781,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); + store->querySubstitutablePathInfos({path}, {}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -784,7 +789,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - WorkerProto::write(*store, wconn, i->second.references); + WorkerProto::write(*store, wconn, i->second.references.possiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -793,21 +798,34 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QuerySubstitutablePathInfos: { SubstitutablePathInfos infos; - StorePathCAMap pathsMap = {}; - if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = WorkerProto::Serialise::read(*store, rconn); - for (auto & path : paths) - pathsMap.emplace(path, std::nullopt); - } else - pathsMap = WorkerProto::Serialise::read(*store, rconn); + StorePathSet paths; + std::set caPaths; + if (GET_PROTOCOL_MINOR(clientVersion) >= 36) { + paths = WorkerProto::Serialise::read(*store, rconn); + caPaths = WorkerProto::Serialise>::read(*store, rconn); + } else if (GET_PROTOCOL_MINOR(clientVersion) >= 22) { + auto pathsMap = WorkerProto::Serialise::read(*store, rconn); + for (auto & [storePath, optCA] : pathsMap) { + if (!optCA) { + paths.insert(storePath); + } else { + caPaths.insert(StorePathDescriptor { + .name = std::string { storePath.name() }, + .info = ContentAddressWithReferences::withoutRefs(*optCA), + }); + } + } + } else { + paths = WorkerProto::Serialise::read(*store, rconn); + } logger->startWork(); - store->querySubstitutablePathInfos(pathsMap, infos); + store->querySubstitutablePathInfos(paths, caPaths, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - WorkerProto::write(*store, wconn, i.second.references); + WorkerProto::write(*store, wconn, i.second.references.possiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; @@ -887,7 +905,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, rconn); + info.setReferencesPossiblyToSelf(WorkerProto::Serialise::read(*store, rconn)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = ContentAddress::parseOpt(readString(from)); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index dc32c3847de..8a8c1346f60 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -38,9 +38,10 @@ std::optional DerivationOutput::path(const Store & store, std::string StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const { - return store.makeFixedOutputPathFromCA( - outputPathName(drvName, outputName), - ContentAddressWithReferences::withoutRefs(ca)); + return store.makeFixedOutputPathFromCA({ + .name = outputPathName(drvName, outputName), + .info = ContentAddressWithReferences::withoutRefs(ca), + }); } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 74d6ed3b518..42a183b6ea0 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -33,7 +33,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store return *uriSchemes().begin(); } - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept override { callback(nullptr); @@ -65,7 +65,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store RepairFlag repair) override { unsupported("addTextToStore"); } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc path, Sink & sink) override { unsupported("narFromPath"); } void queryRealisationUncached(const DrvOutput &, diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index e866aeb42d2..d45a91146d3 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -48,7 +48,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) << printStorePath(path); WorkerProto::write(*this, WorkerProto::WriteConn { .to = teeSink }, - info->references); + info->referencesPossiblyToSelf()); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -84,7 +84,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = references; + info.setReferencesPossiblyToSelf(std::move(references)); info.narSize = saved.s.size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 78b05031ae6..0e7eee502e8 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -159,9 +159,10 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor return *uriSchemes().begin() + "://" + host; } - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept override { + auto path = this->bakeCaIfNeeded(pathOrDesc); try { auto conn(connections->get()); @@ -183,7 +184,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::Serialise::read(*this, *conn); + info->setReferencesPossiblyToSelf(WorkerProto::Serialise::read(*this, *conn)); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -217,7 +218,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, *conn, info.references); + WorkerProto::write(*this, *conn, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize @@ -246,7 +247,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); + WorkerProto::write(*this, *conn, info.referencesPossiblyToSelf()); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -259,8 +260,9 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override { + auto path = this->bakeCaIfNeeded(pathOrDesc); auto conn(connections->get()); conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); @@ -378,7 +380,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor } } - void ensurePath(const StorePath & path) override + void ensurePath(StorePathOrDesc desc) override { unsupported("ensurePath"); } virtual ref getFSAccessor() override @@ -416,19 +418,27 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor out.insert(i); } - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override { auto conn(connections->get()); + StorePathSet paths2; + for (auto & pathOrDesc : paths) + paths2.insert(bakeCaIfNeeded(pathOrDesc)); + conn->to << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, *conn, paths); + WorkerProto::write(*this, *conn, paths2); conn->to.flush(); - return WorkerProto::Serialise::read(*this, *conn); + auto res = WorkerProto::Serialise::read(*this, *conn); + std::set res2; + for (auto & r : res) + res2.insert(r); + return res2; } void connect() override diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index b224fc3e989..fed638a7a5e 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -78,11 +78,12 @@ ref LocalFSStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } -void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) +void LocalFSStore::narFromPath(const StorePathOrDesc pathOrDesc, Sink & sink) { - if (!isValidPath(path)) - throw Error("path '%s' is not valid", printStorePath(path)); - dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); + auto p = this->bakeCaIfNeeded(pathOrDesc); + if (!isValidPath(pathOrDesc)) + throw Error("path '%s' is not valid", printStorePath(p)); + dumpPath(getRealStoreDir() + std::string(printStorePath(p), storeDir.size()), sink); } const std::string LocalFSStore::drvsLogDir = "drvs"; diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 48810950113..e3dafe9783f 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -46,7 +46,7 @@ public: LocalFSStore(const Params & params); - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(StorePathOrDesc path, Sink & sink) override; ref getFSAccessor() override; /** diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17b4ecc7312..a7ef749be45 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,5 +1,6 @@ #include "local-store.hh" #include "globals.hh" +#include "git.hh" #include "archive.hh" #include "pathlocks.hh" #include "worker-protocol.hh" @@ -868,9 +869,10 @@ uint64_t LocalStore::addValidPath(State & state, } -void LocalStore::queryPathInfoUncached(const StorePath & path, +void LocalStore::queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept { + auto path = bakeCaIfNeeded(pathOrDesc); try { callback(retrySQLite>([&]() { auto state(_state.lock()); @@ -922,7 +924,8 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + info->insertReferencePossiblyToSelf( + parseStorePath(useQueryReferences.getStr(0))); return info; } @@ -957,8 +960,9 @@ bool LocalStore::isValidPath_(State & state, const StorePath & path) } -bool LocalStore::isValidPathUncached(const StorePath & path) +bool LocalStore::isValidPathUncached(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); return retrySQLite([&]() { auto state(_state.lock()); return isValidPath_(*state, path); @@ -966,11 +970,11 @@ bool LocalStore::isValidPathUncached(const StorePath & path) } -StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +std::set LocalStore::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { - StorePathSet res; + std::set res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(borrowStorePathOrDesc(i))) res.insert(i); return res; } @@ -1121,7 +1125,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) + for (auto & j : i.referencesPossiblyToSelf()) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1141,7 +1145,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references; + return i == infos.end() ? StorePathSet() : i->second.references.others; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1322,31 +1326,41 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; - if (method == FileIngestionMethod::Recursive) - restorePath(tempPath, bothSource); - else + switch (method) { + case FileIngestionMethod::Flat: writeFile(tempPath, bothSource); + break; + case FileIngestionMethod::Recursive: + restorePath(tempPath, bothSource); + break; + case FileIngestionMethod::Git: + restoreGit(tempPath, bothSource, realStoreDir, storeDir); + break; + } dump.clear(); } auto [hash, size] = hashSink->finish(); - ContentAddressWithReferences desc = FixedOutputInfo { - .method = method, - .hash = hash, - .references = { - .others = references, - // caller is not capable of creating a self-reference, because this is content-addressed without modulus - .self = false, + auto desc = StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + .method = method, + .hash = hash, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, }, }; - auto dstPath = makeFixedOutputPathFromCA(name, desc); + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); - if (repair || !isValidPath(dstPath)) { + if (repair || !isValidPath(desc)) { /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ @@ -1355,7 +1369,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name PathLocks outputLock({realPath}); - if (repair || !isValidPath(dstPath)) { + if (repair || !isValidPath(desc)) { deletePath(realPath); @@ -1364,10 +1378,17 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name if (inMemory) { StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ - if (method == FileIngestionMethod::Recursive) - restorePath(realPath, dumpSource); - else + switch (method) { + case FileIngestionMethod::Flat: writeFile(realPath, dumpSource); + break; + case FileIngestionMethod::Recursive: + restorePath(realPath, dumpSource); + break; + case FileIngestionMethod::Git: + restoreGit(realPath, dumpSource, realStoreDir, storeDir); + break; + } } else { /* Move the temporary path we restored above. */ moveFile(tempPath, realPath); @@ -1386,12 +1407,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name optimisePath(realPath, repair); - ValidPathInfo info { - *this, - name, - std::move(desc), - narHash.first - }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; registerValidPath(info); } @@ -1440,7 +1456,8 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - info.references = references; + // No self reference allowed with text-hashing + info.references.others = references; info.ca = { .method = TextIngestionMethod {}, .hash = hash, @@ -1866,25 +1883,37 @@ ContentAddress LocalStore::hashCAPath( const std::string_view pathHash ) { - HashModuloSink caSink ( hashType, std::string(pathHash) ); + Hash hash { htSHA256 }; // throwaway def to appease C++ std::visit(overloaded { [&](const TextIngestionMethod &) { + HashModuloSink caSink ( hashType, std::string(pathHash) ); readFile(path, caSink); + hash = caSink.finish().first; }, [&](const FileIngestionMethod & m2) { switch (m2) { - case FileIngestionMethod::Recursive: + case FileIngestionMethod::Recursive: { + HashModuloSink caSink ( hashType, std::string(pathHash) ); dumpPath(path, caSink); + hash = caSink.finish().first; break; - case FileIngestionMethod::Flat: + } + case FileIngestionMethod::Flat: { + HashModuloSink caSink ( hashType, std::string(pathHash) ); readFile(path, caSink); + hash = caSink.finish().first; break; } + case FileIngestionMethod::Git: { + hash = dumpGitHash(hashType, path); + break; + } + } }, }, method.raw); return ContentAddress { .method = method, - .hash = caSink.finish().first, + .hash = std::move(hash), }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e97195f5b88..41e48f6600f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -152,14 +152,14 @@ public: std::string getUri() override; - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc, Callback> callback) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 253609ed2a9..1fe8be40b36 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -28,15 +28,13 @@ std::map makeContentAddressed( StringMap rewrites; StoreReferences refs; - for (auto & ref : oldInfo->references) { - if (ref == path) - refs.self = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + refs.self = oldInfo->references.self; + for (auto & ref : oldInfo->references.others) { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) { + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); refs.others.insert(std::move(replacement)); } } @@ -50,11 +48,13 @@ std::map makeContentAddressed( ValidPathInfo info { dstStore, - path.name(), - FixedOutputInfo { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - .references = std::move(refs), + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, + .references = std::move(refs), + }, }, Hash::dummy, }; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c043b9b9305..249a9ca5b65 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -40,9 +40,8 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto& ref : info->references) - if (ref != path) - res.insert(ref); + for (auto & ref : info->references.others) + res.insert(ref); if (includeOutputs && path.isDerivation()) for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) @@ -83,15 +82,18 @@ void Store::computeFSClosure(const StorePath & startPath, } -const ContentAddress * getDerivationCA(const BasicDerivation & drv) +std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out == drv.outputs.end()) - return nullptr; + return std::nullopt; if (auto dof = std::get_if(&out->second.raw)) { - return &dof->ca; + return StorePathDescriptor { + .name = drv.name, + .info = ContentAddressWithReferences::withoutRefs(dof->ca), + }; } - return nullptr; + return std::nullopt; } void Store::queryMissing(const std::vector & targets, @@ -141,13 +143,11 @@ void Store::queryMissing(const std::vector & targets, if (drvState_->lock()->done) return; SubstitutablePathInfos infos; - auto * cap = getDerivationCA(*drv); - querySubstitutablePathInfos({ - { - outPath, - cap ? std::optional { *cap } : std::nullopt, - }, - }, infos); + auto caOpt = getDerivationCA(*drv); + if (caOpt) + querySubstitutablePathInfos({}, { *std::move(caOpt) }, infos); + else + querySubstitutablePathInfos({outPath}, {}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -251,7 +251,7 @@ void Store::queryMissing(const std::vector & targets, if (isValidPath(bo.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos); + querySubstitutablePathInfos({bo.path}, {}, infos); if (infos.empty()) { auto state(state_.lock()); @@ -269,7 +269,7 @@ void Store::queryMissing(const std::vector & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references) + for (auto & ref : info->second.references.others) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -287,7 +287,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references; + return queryPathInfo(path)->references.others; } catch (InvalidPath &) { return StorePathSet(); } @@ -300,6 +300,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } + std::map drvOutputReferences( const std::set & inputRealisations, const StorePathSet & pathReferences) @@ -343,7 +344,7 @@ std::map drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); } OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f0dfcb19b77..3cdd6b64513 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -69,9 +69,10 @@ struct NarAccessor : public FSAccessor createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); } - void createRegularFile(const Path & path) override + void createRegularFile(const Path & path, bool executable = false) override { createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + if (executable) isExecutable(); } void closeRegularFile() override diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c7176d30f7a..3114ac972fb 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -268,7 +268,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath(r)); + narInfo->insertReferencePossiblyToSelf(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d17253741a6..c035de748fb 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -66,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) throw corrupt("extra References"); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 5dbdafac3eb..b9952303272 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,8 +17,8 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) - : ValidPathInfo(store, std::move(name), std::move(ca), narHash) + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ccb57104f91..97f0e4bbfb1 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -14,7 +14,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); } @@ -23,46 +23,40 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::contentAddressWithReferences() const +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const { if (! ca) return std::nullopt; - return std::visit(overloaded { - [&](const TextIngestionMethod &) -> ContentAddressWithReferences { - assert(references.count(path) == 0); - return TextInfo { - .hash = ca->hash, - .references = references, - }; - }, - [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return FixedOutputInfo { - .method = m2, - .hash = ca->hash, - .references = { - .others = std::move(refs), - .self = hasSelfReference, - }, - }; - }, - }, ca->method.raw); + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { + assert(!references.self); + return TextInfo { + .hash = ca->hash, + .references = references.others, + }; + }, + [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { + return FixedOutputInfo { + .method = m2, + .hash = ca->hash, + .references = references, + }; + }, + }, ca->method.raw), + }; } bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = contentAddressWithReferences(); + auto fullCaOpt = fullStorePathDescriptorOpt(); if (! fullCaOpt) return false; - auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -94,7 +88,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : references) + for (auto & r : referencesPossiblyToSelf()) refs.push_back(std::string(r.to_string())); return refs; } @@ -102,30 +96,46 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - std::string_view name, - ContentAddressWithReferences && ca, + StorePathDescriptor && info, Hash narHash) - : path(store.makeFixedOutputPathFromCA(name, ca)) + : path(store.makeFixedOutputPathFromCA(info)) , narHash(narHash) { std::visit(overloaded { [this](TextInfo && ti) { - this->references = std::move(ti.references); + this->references = { + .others = std::move(ti.references), + .self = false, + }; this->ca = ContentAddress { .method = TextIngestionMethod {}, .hash = std::move(ti.hash), }; }, [this](FixedOutputInfo && foi) { - this->references = std::move(foi.references.others); - if (foi.references.self) - this->references.insert(path); + this->references = std::move(foi.references); this->ca = ContentAddress { .method = std::move(foi.method), .hash = std::move(foi.hash), }; }, - }, std::move(ca).raw); + }, std::move(info.info).raw); +} + + +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return references.possiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return references.insertPossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return references.setPossiblyToSelf(path, std::move(refs)); } @@ -140,8 +150,8 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = source }); + info.setReferencesPossiblyToSelf(WorkerProto::Serialise::read(store, + WorkerProto::ReadConn { .from = source })); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -164,7 +174,7 @@ void ValidPathInfo::write( << narHash.to_string(Base16, false); WorkerProto::write(store, WorkerProto::WriteConn { .to = sink }, - references); + referencesPossiblyToSelf()); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 22152362206..18baae9eff1 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -18,7 +18,7 @@ class Store; struct SubstitutablePathInfo { std::optional deriver; - StorePathSet references; + StoreReferences references; /** * 0 = unknown or inapplicable */ @@ -31,7 +31,6 @@ struct SubstitutablePathInfo typedef std::map SubstitutablePathInfos; - struct ValidPathInfo { StorePath path; @@ -40,7 +39,7 @@ struct ValidPathInfo * \todo document this */ Hash narHash; - StorePathSet references; + StoreReferences references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -93,17 +92,23 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); /** - * @return The `ContentAddressWithReferences` that determines the + * @return The `StorePathDescriptor` that determines the * store path for a content-addressed store object, `std::nullopt` * for an input-addressed store object. */ - std::optional contentAddressWithReferences() const; + std::optional fullStorePathDescriptorOpt() const; /** * @return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; + void insertReferencePossiblyToSelf(StorePath && ref); + void setReferencesPossiblyToSelf(StorePathSet && refs); + static const size_t maxSigs = std::numeric_limits::max(); /** @@ -126,7 +131,7 @@ struct ValidPathInfo ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const Store & store, - std::string_view name, ContentAddressWithReferences && ca, Hash narHash); + StorePathDescriptor && ca, Hash narHash); virtual ~ValidPathInfo() { } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 58f72beb90c..150fc60ff0d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -13,6 +13,7 @@ #include "derivations.hh" #include "pool.hh" #include "finally.hh" +#include "git.hh" #include "logging.hh" #include "callback.hh" #include "filetransfer.hh" @@ -187,8 +188,9 @@ void RemoteStore::setOptions() setOptions(*(getConnection().handle)); } -bool RemoteStore::isValidPathUncached(const StorePath & path) +bool RemoteStore::isValidPathUncached(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(getConnection()); conn->to << WorkerProto::Op::IsValidPath << printStorePath(path); conn.processStderr(); @@ -196,22 +198,31 @@ bool RemoteStore::isValidPathUncached(const StorePath & path) } -StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +std::set RemoteStore::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - StorePathSet res; + std::set res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(borrowStorePathOrDesc(i))) + res.insert(i); return res; } else { conn->to << WorkerProto::Op::QueryValidPaths; - WorkerProto::write(*this, *conn, paths); + StorePathSet paths2; + for (auto & pathOrDesc : paths) + paths2.insert(bakeCaIfNeeded(pathOrDesc)); + // FIXME make new version to take advantage of desc case + WorkerProto::write(*this, *conn, paths2); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { conn->to << (settings.buildersUseSubstitutes ? 1 : 0); } conn.processStderr(); - return WorkerProto::Serialise::read(*this, *conn); + auto res = WorkerProto::Serialise::read(*this, *conn); + std::set res2; + for (auto & r : res) + res2.insert(r); + return res2; } } @@ -245,58 +256,76 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { - if (pathsMap.empty()) return; + if (paths.empty() && caPaths.empty()) return; auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + auto combineSet = [&]() { + std::set combined { paths }; + for (auto & ca : caPaths) + combined.insert(makeFixedOutputPathFromCA(ca)); + return combined; + }; - for (auto & i : pathsMap) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + for (auto & path : combineSet()) { SubstitutablePathInfo info; - conn->to << WorkerProto::Op::QuerySubstitutablePathInfo << printStorePath(i.first); + conn->to << WorkerProto::Op::QuerySubstitutablePathInfo << printStorePath(path); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*this, *conn); + info.references.setPossiblyToSelf(path, WorkerProto::Serialise::read(*this, *conn)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.first, std::move(info)); + infos.insert_or_assign(path, std::move(info)); } } else { conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { - StorePathSet paths; - for (auto & path : pathsMap) - paths.insert(path.first); + WorkerProto::write(*this, *conn, combineSet()); + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 36) { + WorkerProto::StorePathCAMap pathMap; + for (auto & path : paths) + pathMap[path]; + for (auto & desc : caPaths) { + auto path = makeFixedOutputPathFromCA(desc); + pathMap[path] = { + .method = desc.info.getMethod(), + .hash = desc.info.getHash(), + }; + } + WorkerProto::write(*this, *conn, pathMap); + } else { WorkerProto::write(*this, *conn, paths); - } else - WorkerProto::write(*this, *conn, pathsMap); + WorkerProto::write(*this, *conn, caPaths); + } conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto path = parseStorePath(readString(conn->from)); + SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*this, *conn); + info.references.setPossiblyToSelf(path, WorkerProto::Serialise::read(*this, *conn)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } - } } -void RemoteStore::queryPathInfoUncached(const StorePath & path, +void RemoteStore::queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept { + auto path = bakeCaIfNeeded(pathOrDesc); try { std::shared_ptr info; { @@ -506,7 +535,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); + WorkerProto::write(*this, *conn, info.referencesPossiblyToSelf()); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -525,7 +554,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, *conn, info.references); + WorkerProto::write(*this, *conn, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -815,8 +844,9 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD } -void RemoteStore::ensurePath(const StorePath & path) +void RemoteStore::ensurePath(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(getConnection()); conn->to << WorkerProto::Op::EnsurePath << printStorePath(path); conn.processStderr(); @@ -979,8 +1009,9 @@ RemoteStore::Connection::~Connection() } } -void RemoteStore::narFromPath(const StorePath & path, Sink & sink) +void RemoteStore::narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(connections->get()); conn->to << WorkerProto::Op::NarFromPath << printStorePath(path); conn->processStderr(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a1ae82a0f3f..8b2483d8861 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -46,14 +46,14 @@ public: /* Implementations of abstract store API methods. */ - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc, Callback> callback) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; @@ -67,7 +67,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; /** @@ -122,7 +123,7 @@ public: BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const StorePath & path) override; + void ensurePath(StorePathOrDesc path) override; void addTempRoot(const StorePath & path) override; @@ -187,7 +188,7 @@ protected: virtual ref getFSAccessor() override; - virtual void narFromPath(const StorePath & path, Sink & sink) override; + virtual void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override; private: diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d2fc6abafe4..a2d743b3946 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -309,8 +309,9 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual fetches the .narinfo file, rather than first checking for its existence via a HEAD request. Since .narinfos are small, doing a GET is unlikely to be slower than HEAD. */ - bool isValidPathUncached(const StorePath & storePath) override + bool isValidPathUncached(StorePathOrDesc storePathOrDesc) override { + auto storePath = bakeCaIfNeeded(storePathOrDesc); try { queryPathInfo(storePath); return true; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28689e100e2..1bbb4111667 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -10,6 +10,7 @@ #include "references.hh" #include "archive.hh" #include "callback.hh" +#include "git.hh" #include "remote-store.hh" #include @@ -114,8 +115,8 @@ StorePath Store::followLinksToStorePath(std::string_view path) const for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:::", where - = "r:" for recursive (path) hashes, or "" for flat - (file) hashes + = "r:" for recursive (path) hashes, "git:" for git + paths, or "" for flat (file) hashes = "md5", "sha1" or "sha256" = base-16 representation of the path or flat hash of the contents of the path (or expected contents of the @@ -181,9 +182,37 @@ static std::string makeType( return std::move(type); } +StorePath Store::bakeCaIfNeeded(const OwnedStorePathOrDesc & path) const +{ + return bakeCaIfNeeded(borrowStorePathOrDesc(path)); +} + +StorePath Store::bakeCaIfNeeded(StorePathOrDesc path) const +{ + return std::visit(overloaded { + [this](std::reference_wrapper storePath) { + return StorePath {storePath}; + }, + [this](std::reference_wrapper ca) { + return makeFixedOutputPathFromCA(ca); + }, + }, path); +} + +StorePathOrDesc borrowStorePathOrDesc(const OwnedStorePathOrDesc & storePathOrDesc) { + // Can't use std::visit as it would copy :( + if (auto p = std::get_if(&storePathOrDesc)) + return *p; + if (auto p = std::get_if(&storePathOrDesc)) + return *p; + abort(); +} StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { + if (info.method == FileIngestionMethod::Git && info.hash.type != htSHA1) + throw Error("Git file ingestion must use sha1 hash"); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { @@ -202,35 +231,45 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", StoreReferences { - .others = info.references, - .self = false, - }), + makeType(*this, "text", StoreReferences { info.references }), info.hash, name); } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { [&](const TextInfo & ti) { - return makeTextPath(name, ti); + return makeTextPath(desc.name, ti); }, [&](const FixedOutputInfo & foi) { - return makeFixedOutputPath(name, foi); + return makeFixedOutputPath(desc.name, foi); } - }, ca.raw); + }, desc.info.raw); } std::pair Store::computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const { - Hash h = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath, filter).first - : hashFile(hashAlgo, srcPath); + Hash h { htSHA256 }; // throwaway def to appease C++ + switch (method) { + case FileIngestionMethod::Recursive: { + h = hashPath(hashAlgo, srcPath, filter).first; + break; + } + case FileIngestionMethod::Git: { + h = hashGit(hashAlgo, srcPath, filter).first; + break; + } + case FileIngestionMethod::Flat: { + h = hashFile(hashAlgo, srcPath); + break; + } + } + FixedOutputInfo caInfo { .method = method, .hash = h, @@ -263,10 +302,28 @@ StorePath Store::addToStore( { Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { - if (method == FileIngestionMethod::Recursive) + switch (method) { + case FileIngestionMethod::Recursive: { dumpPath(srcPath, sink, filter); - else + break; + } + case FileIngestionMethod::Git: { + // recursively add to store if path is a directory + struct stat st; + if (lstat(srcPath.c_str(), &st)) + throw SysError("getting attributes of path '%1%'", srcPath); + if (S_ISDIR(st.st_mode)) + for (auto & i : readDirectory(srcPath)) + addToStore("git", srcPath + "/" + i.name, method, hashAlgo, filter, repair); + + dumpGit(hashAlgo, srcPath, sink, filter); + break; + } + case FileIngestionMethod::Flat: { readFile(srcPath, sink); + break; + } + } }); return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); } @@ -313,7 +370,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references; + return info.references.others; }, [&](const StorePath & path) { @@ -430,6 +487,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 ? narHash + : method == FileIngestionMethod::Git + ? hashGit(hashAlgo, srcPath).first : caHashSink.finish().first; if (expectedCAHash && expectedCAHash != hash) @@ -437,11 +496,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, ValidPathInfo info { *this, - name, - FixedOutputInfo { - .method = method, - .hash = hash, - .references = {}, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + .method = method, + .hash = hash, + .references = {}, + }, }, narHash, }; @@ -552,58 +613,66 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) } -void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +void Store::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - for (auto & path : paths) { - if (infos.count(path.first)) - // Choose first succeeding substituter. - continue; - - auto subPath(path.first); - - // Recompute store path so that we can use a different store root. - if (path.second) { - subPath = makeFixedOutputPathFromCA( - path.first.name(), - ContentAddressWithReferences::withoutRefs(*path.second)); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; + auto query = [&](auto & sub, const StorePath & localPath, const StorePath & subPath) { + if (infos.count(subPath)) + // Choose first succeeding substituter. + return; - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo{ - .deriver = info->deriver, - .references = info->references, - .downloadSize = narInfo ? narInfo->fileSize : 0, - .narSize = info->narSize, - }); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + return; + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + infos.insert_or_assign(localPath, SubstitutablePathInfo { + info->deriver, + info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize, + }); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + }; + + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (sub->storeDir != storeDir) continue; + query(sub, path, path); + } + for (auto & ca : caPaths) { + // TODO Deal with references: either disallow, or require the + // store path lengths be the same and rewrite strings. + auto localPath = makeFixedOutputPathFromCA(ca); + auto subPath = sub->makeFixedOutputPathFromCA(ca); + if (sub->storeDir == storeDir) + assert(localPath == subPath); + if (localPath != subPath) + // TODO print CA too + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(localPath), sub->printStorePath(subPath), sub->getUri()); + query(sub, localPath, subPath); } } } -bool Store::isValidPath(const StorePath & storePath) +bool Store::isValidPath(StorePathOrDesc storePathOrDesc) { + auto storePath = bakeCaIfNeeded(storePathOrDesc); + { auto state_(state.lock()); auto res = state_->pathInfoCache.get(std::string(storePath.to_string())); @@ -636,7 +705,7 @@ bool Store::isValidPath(const StorePath & storePath) /* Default implementation for stores that only implement queryPathInfoUncached(). */ -bool Store::isValidPathUncached(const StorePath & path) +bool Store::isValidPathUncached(StorePathOrDesc path) { try { queryPathInfo(path); @@ -647,7 +716,7 @@ bool Store::isValidPathUncached(const StorePath & path) } -ref Store::queryPathInfo(const StorePath & storePath) +ref Store::queryPathInfo(StorePathOrDesc storePath) { std::promise> promise; @@ -672,9 +741,11 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual) } -void Store::queryPathInfo(const StorePath & storePath, +void Store::queryPathInfo(StorePathOrDesc pathOrCa, Callback> callback) noexcept { + auto storePath = bakeCaIfNeeded(pathOrCa); + auto hashPart = std::string(storePath.hashPart()); try { @@ -708,7 +779,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); - queryPathInfoUncached(storePath, + queryPathInfoUncached(pathOrCa, {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { @@ -824,22 +895,38 @@ void Store::substitutePaths(const StorePathSet & paths) StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + std::set paths2; + for (auto & p : paths) + paths2.insert(p); + auto res = queryValidPaths(paths2, maybeSubstitute); + StorePathSet res2; + for (auto & r : res) { + auto p = std::get_if(&r); + assert(p); + res2.insert(*p); + } + return res2; +} + +std::set Store::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { struct State { size_t left; - StorePathSet valid; + std::set valid; std::exception_ptr exc; }; - Sync state_(State{paths.size(), StorePathSet()}); + Sync state_(State{paths.size(), {}}); std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const StorePath & path) { + auto doQuery = [&](const OwnedStorePathOrDesc & path) { checkInterrupt(); - queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { + auto path2 = borrowStorePathOrDesc(path); + queryPathInfo(path2, {[path, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); @@ -893,7 +980,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += fmt("%1%\n", info->references.size()); - for (auto & j : info->references) + for (auto & j : info->referencesPossiblyToSelf()) s += printStorePath(j) + "\n"; } @@ -956,7 +1043,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1042,6 +1129,7 @@ static std::string makeCopyPathMessage( std::string_view dstUri, std::string_view storePath) { + // FIXME Use CA when we have it in messages below return srcUri == "local" || srcUri == "daemon" ? fmt("copying path '%s' to '%s'", storePath, dstUri) : dstUri == "local" || dstUri == "daemon" @@ -1049,17 +1137,19 @@ static std::string makeCopyPathMessage( : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri); } - void copyStorePath( Store & srcStore, Store & dstStore, - const StorePath & storePath, + StorePathOrDesc storePath, RepairFlag repair, CheckSigsFlag checkSigs) { + + auto actualStorePath = srcStore.bakeCaIfNeeded(storePath); + auto srcUri = srcStore.getUri(); auto dstUri = dstStore.getUri(); - auto storePathS = srcStore.printStorePath(storePath); + auto storePathS = srcStore.printStorePath(actualStorePath); Activity act(*logger, lvlInfo, actCopyPath, makeCopyPathMessage(srcUri, dstUri, storePathS), {storePathS, srcUri, dstUri}); @@ -1070,14 +1160,21 @@ void copyStorePath( uint64_t total = 0; // recompute store path on the chance dstStore does it differently - if (info->ca && info->references.empty()) { - auto info2 = make_ref(*info); - info2->path = dstStore.makeFixedOutputPathFromCA( - info->path.name(), - info->contentAddressWithReferences().value()); - if (dstStore.storeDir == srcStore.storeDir) - assert(info->path == info2->path); - info = info2; + if (auto p = std::get_if>(&storePath)) { + auto ca = static_cast(*p); + // { + // ValidPathInfo srcInfoCA { *srcStore, StorePathDescriptor { ca } }; + // assert((StoreReferences &)(*info) == (StoreReferences &)srcInfoCA); + // } + if (info->references.empty()) { + auto info2 = make_ref(*info); + ValidPathInfo dstInfoCA { dstStore, StorePathDescriptor { ca }, info->narHash }; + if (dstStore.storeDir == srcStore.storeDir) + assert(info2->path == info2->path); + info2->path = std::move(dstInfoCA.path); + info2->ca = std::move(dstInfoCA.ca); + info = info2; + } } if (info->ultimate) { @@ -1094,14 +1191,14 @@ void copyStorePath( TeeSink tee { sink, progressSink }; srcStore.narFromPath(storePath, tee); }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(storePath), srcStore.getUri()); + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(actualStorePath), srcStore.getUri()); }); dstStore.addToStore(*info, *source, repair, checkSigs); } -std::map copyPaths( +void copyPaths( Store & srcStore, Store & dstStore, const RealisedPath::Set & paths, @@ -1118,7 +1215,7 @@ std::map copyPaths( toplevelRealisations.insert(*realisation); } } - auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); + copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); ThreadPool pool; @@ -1151,11 +1248,10 @@ std::map copyPaths( else throw; } - - return pathsMap; } -std::map copyPaths( + +void copyPaths( Store & srcStore, Store & dstStore, const StorePathSet & storePaths, @@ -1171,74 +1267,149 @@ std::map copyPaths( Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + uint64_t total = 0; + + auto sorted = srcStore.topoSortPaths(missing); + std::reverse(sorted.begin(), sorted.end()); + + auto source = sinkToSource([&](Sink & sink) { + sink << sorted.size(); + for (auto & storePath : sorted) { + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(storePath); + Activity act(*logger, lvlInfo, actCopyPath, + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); + PushActivity pact(act.id); + + auto info = srcStore.queryPathInfo(storePath); + info->write(sink, srcStore, 16); + + LambdaSink progressSink([&](std::string_view data) { + total += data.size(); + act.progress(total, info->narSize); + }); + TeeSink tee { sink, progressSink }; + + srcStore.narFromPath(storePath, tee); + } + }); + + dstStore.addMultipleToStore(*source, repair, checkSigs); +} + + +void copyPaths( + Store & srcStore, + Store & dstStore, + const std::set & storePaths, + RepairFlag repair, + CheckSigsFlag checkSigs, + SubstituteFlag substitute) +{ + auto valid = dstStore.queryValidPaths(storePaths, substitute); + + std::set missing; + + // Filter to just the ones which are missing. + + // While doing so, if we can "upgrade" the store path into a store + // path descriptor because it is content-addressed, do that. + for (auto & path : storePaths) { + if (valid.count(path)) continue; + + auto pathDescOpt = [&]() -> std::optional> { + auto storePathP = std::get_if(&path); + // Already store path descriptor + if (!storePathP) return std::nullopt; + auto & storePath = *storePathP; + + auto info = srcStore.queryPathInfo(borrowStorePathOrDesc(path)); + + auto descOpt = info->fullStorePathDescriptorOpt(); + if (!descOpt) return std::nullopt; + + return { { storePath, *std::move(descOpt) } }; + }(); + + if (pathDescOpt) { + auto [storePath, desc] = *std::move(pathDescOpt); + + debug("found CA description '%s' for path '%s'", + renderStorePathDescriptor(desc), + srcStore.printStorePath(storePath)); + + missing.insert(std::move(desc)); + } else { + missing.insert(path); + } + } + + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + + uint64_t total = 0; + // In the general case, `addMultipleToStore` requires a sorted list of // store paths to add, so sort them right now - auto sortedMissing = srcStore.topoSortPaths(missing); - std::reverse(sortedMissing.begin(), sortedMissing.end()); - std::map pathsMap; - for (auto & path : storePaths) - pathsMap.insert_or_assign(path, path); + // Only paths, no descriptors + StorePathSet missingPaths; + // Map from paths back to original + std::map rich; - Store::PathsSource pathsToCopy; + // Build + for (auto & pathOrDesc : missing) { + auto path = srcStore.bakeCaIfNeeded(pathOrDesc); + rich.insert_or_assign(path, &pathOrDesc); + } - auto computeStorePathForDst = [&](const ValidPathInfo & currentPathInfo) -> StorePath { - auto storePathForSrc = currentPathInfo.path; - auto storePathForDst = storePathForSrc; - if (currentPathInfo.ca && currentPathInfo.references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA( - currentPathInfo.path.name(), - currentPathInfo.contentAddressWithReferences().value()); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePathForSrc); - if (storePathForDst != storePathForSrc) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePathForSrc), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - return storePathForDst; - }; + // Sort paths + auto sortedMissingPaths = srcStore.topoSortPaths(missingPaths); + std::reverse(sortedMissingPaths.begin(), sortedMissingPaths.end()); - // total is accessed by each copy, which are each handled in separate threads - std::atomic total = 0; + // Sorted originals + std::vector sortedMissing; + for (auto & path : sortedMissingPaths) + sortedMissing.push_back(rich.at(path)); - for (auto & missingPath : sortedMissing) { - auto info = srcStore.queryPathInfo(missingPath); + auto source = sinkToSource([&](Sink & sink) { + sink << sortedMissing.size(); + for (auto * missingPathP : sortedMissing) { + auto & missingPath = *missingPathP; - auto storePathForDst = computeStorePathForDst(*info); - pathsMap.insert_or_assign(missingPath, storePathForDst); + // For destination store + auto info = *srcStore.queryPathInfo(borrowStorePathOrDesc(missingPath)); - ValidPathInfo infoForDst = *info; - infoForDst.path = storePathForDst; + if (auto * pDesc = std::get_if(&missingPath)) + info.path = dstStore.makeFixedOutputPathFromCA(*pDesc); - auto source = sinkToSource([&](Sink & sink) { // We can reasonably assume that the copy will happen whenever we // read the path, so log something about that at that point auto srcUri = srcStore.getUri(); auto dstUri = dstStore.getUri(); - auto storePathS = srcStore.printStorePath(missingPath); + auto storePathS = srcStore.printStorePath(info.path); Activity act(*logger, lvlInfo, actCopyPath, makeCopyPathMessage(srcUri, dstUri, storePathS), {storePathS, srcUri, dstUri}); PushActivity pact(act.id); + info.write(sink, srcStore, 16); + LambdaSink progressSink([&](std::string_view data) { total += data.size(); - act.progress(total, info->narSize); + act.progress(total, info.narSize); }); TeeSink tee { sink, progressSink }; - srcStore.narFromPath(missingPath, tee); - }); - pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); - } - - dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); + srcStore.narFromPath(borrowStorePathOrDesc(missingPath), tee); + } + }); - return pathsMap; + dstStore.addMultipleToStore(*source, repair, checkSigs); } + void copyClosure( Store & srcStore, Store & dstStore, @@ -1295,7 +1466,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!n) throw Error("number expected"); while ((*n)--) { getline(str, s); - info.references.insert(store.parseStorePath(s)); + info.insertReferencePossiblyToSelf(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional(std::move(info)); @@ -1318,7 +1489,6 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } - Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index da1a3eefbbc..95e402d6131 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -95,7 +95,15 @@ struct BuildResult; struct KeyedBuildResult; -typedef std::map> StorePathCAMap; +/* Useful for many store functions which can take advantage of content + addresses or work with regular store paths */ +typedef std::variant OwnedStorePathOrDesc; +typedef std::variant< + std::reference_wrapper, + std::reference_wrapper +> StorePathOrDesc; + +StorePathOrDesc borrowStorePathOrDesc(const OwnedStorePathOrDesc &); struct StoreConfig : public Config { @@ -286,7 +294,10 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; + + StorePath bakeCaIfNeeded(StorePathOrDesc path) const; + StorePath bakeCaIfNeeded(const OwnedStorePathOrDesc & path) const; /** * Preparatory part of addToStore(). @@ -322,11 +333,11 @@ public: /** * Check whether a path is valid. */ - bool isValidPath(const StorePath & path); + bool isValidPath(StorePathOrDesc desc); protected: - virtual bool isValidPathUncached(const StorePath & path); + virtual bool isValidPathUncached(StorePathOrDesc desc); public: @@ -341,6 +352,8 @@ public: * Query which of the given paths is valid. Optionally, try to * substitute missing paths. */ + virtual std::set queryValidPaths(const std::set & paths, + SubstituteFlag maybeSubstitute = NoSubstitute); virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); @@ -361,12 +374,12 @@ public: * Query information about a valid path. It is permitted to omit * the name part of the store path. */ - ref queryPathInfo(const StorePath & path); + ref queryPathInfo(StorePathOrDesc path); /** * Asynchronous version of queryPathInfo(). */ - void queryPathInfo(const StorePath & path, + void queryPathInfo(StorePathOrDesc path, Callback> callback) noexcept; /** @@ -404,7 +417,7 @@ public: protected: - virtual void queryPathInfoUncached(const StorePath & path, + virtual void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept = 0; virtual void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept = 0; @@ -477,7 +490,8 @@ public: * If a path does not have substitute info, it's omitted from the * resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, + virtual void querySubstitutablePathInfos(const StorePathSet & paths, + const std::set & caPaths, SubstitutablePathInfos & infos); /** @@ -573,7 +587,7 @@ public: /** * Write a NAR dump of a store path. */ - virtual void narFromPath(const StorePath & path, Sink & sink) = 0; + virtual void narFromPath(StorePathOrDesc desc, Sink & sink) = 0; /** * For each path, if it's a derivation, build it. Building a @@ -644,7 +658,7 @@ public: * may be made valid by running a substitute (if defined for the * path). */ - virtual void ensurePath(const StorePath & path); + virtual void ensurePath(StorePathOrDesc desc); /** * Add a store path as a temporary root of the garbage collector. @@ -889,7 +903,7 @@ protected: void copyStorePath( Store & srcStore, Store & dstStore, - const StorePath & storePath, + StorePathOrDesc storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); @@ -899,19 +913,24 @@ void copyStorePath( * in parallel. They are copied in a topologically sorted order (i.e. if * A is a reference of B, then A is copied before B), but the set of * store paths is not automatically closed; use copyClosure() for that. - * - * @return a map of what each path was copied to the dstStore as. */ -std::map copyPaths( +void copyPaths( Store & srcStore, Store & dstStore, const RealisedPath::Set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -std::map copyPaths( +void copyPaths( Store & srcStore, Store & dstStore, - const StorePathSet & paths, + const std::set & storePaths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); + +void copyPaths( + Store & srcStore, Store & dstStore, + const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -1048,7 +1067,7 @@ std::optional decodeValidPathInfo( */ std::pair splitUriAndParams(const std::string & uri); -const ContentAddress * getDerivationCA(const BasicDerivation & drv); +std::optional getDerivationCA(const BasicDerivation & drv); std::map drvOutputReferences( Store & store, diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index cdb28a001cf..057127e5fda 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -38,7 +38,7 @@ public: ref getFSAccessor() override { return LocalFSStore::getFSAccessor(); } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } /** diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a23130743ee..6d11b0bc0d2 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -79,6 +79,17 @@ void WorkerProto::Serialise::write(const Store & store, WorkerPr } +StorePathDescriptor WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +{ + return parseStorePathDescriptor(readString(conn.from)); +} + +void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const StorePathDescriptor & ca) +{ + conn.to << renderStorePathDescriptor(ca); +} + + DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ff762c924ab..63754364ef2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 35) +#define PROTOCOL_VERSION (1 << 8 | 36) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -111,6 +111,11 @@ struct WorkerProto { WorkerProto::Serialise::write(store, conn, t); } + + /** + * For an older version of the protocol + */ + typedef std::map> StorePathCAMap; }; enum struct WorkerProto::Op : uint64_t @@ -204,6 +209,8 @@ MAKE_WORKER_PROTO(StorePath); template<> MAKE_WORKER_PROTO(ContentAddress); template<> +MAKE_WORKER_PROTO(StorePathDescriptor); +template<> MAKE_WORKER_PROTO(DerivedPath); template<> MAKE_WORKER_PROTO(Realisation); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 268a798d900..4e3ed4a7534 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -302,71 +302,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - - void restorePath(const Path & path, Source & source) { RestoreSink sink; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 2cf164a417d..36e6fc41fb0 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -3,7 +3,7 @@ #include "types.hh" #include "serialise.hh" - +#include "fs-sink.hh" namespace nix { @@ -72,22 +72,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -/** - * \todo Fix this API, it sucks. - */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; - virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; - - virtual void createSymlink(const Path & path, const std::string & target) { }; -}; - /** * If the NAR archive contains a single file at top-level, then save * the contents of the file to `s`. Otherwise barf. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 782331283ca..4f2ae03c46c 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -70,6 +70,14 @@ constexpr std::array xpFeatureDetails = {{ [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. )", }, + { + .tag = Xp::GitHashing, + .name = "git-hashing", + .description = R"( + Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm. + These store objects will not be understandable by older versions of Nix. + )", + }, { .tag = Xp::RecursiveNix, .name = "recursive-nix", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index add592ae624..f0e09a41f12 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -21,6 +21,7 @@ enum struct ExperimentalFeature ImpureDerivations, Flakes, NixCommand, + GitHashing, RecursiveNix, NoUrlLiterals, FetchClosure, diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc new file mode 100644 index 00000000000..978b676ba4e --- /dev/null +++ b/src/libutil/fs-sink.cc @@ -0,0 +1,104 @@ +#include + +#include "config.hh" +#include "fs-sink.hh" + +namespace nix { + + +struct RestoreSinkSettings : Config +{ + Setting preallocateContents{this, true, "preallocate-contents", + "Whether to preallocate files when writing objects with known size."}; +}; + +static RestoreSinkSettings restoreSinkSettings; + +static GlobalConfig::Register r1(&restoreSinkSettings); + + +void RestoreSink::createDirectory(const Path & path) +{ + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError("creating directory '%1%'", p); +}; + +void RestoreSink::createRegularFile(const Path & path, bool executable) +{ + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, executable ? 0777 : 0666); + if (!fd) throw SysError("creating file '%1%'", p); +} + +void RestoreSink::closeRegularFile() +{ + /* Call close explicitly to make sure the error is checked */ + fd.close(); +} + +void RestoreSink::isExecutable() +{ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); +} + +void RestoreSink::preallocateContents(uint64_t len) +{ + if (!restoreSinkSettings.preallocateContents) + return; + +#ifdef HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif +} + +void RestoreSink::receiveContents(std::string_view data) +{ + writeFull(fd.get(), data); +} + +void RestoreSink::createSymlink(const Path & path, const std::string & target) +{ + Path p = dstPath + path; + nix::createSymlink(target, p); +} + +void RestoreSink::copyFile(const Path & source) +{ + FdSink sink(fd.get()); + readFile(source, sink); +} + +void RestoreSink::copyDirectory(const Path & source, const Path & destination) +{ + Path p = dstPath + destination; + createDirectory(destination); + for (auto & i : readDirectory(source)) { + struct stat st; + Path entry = source + "/" + i.name; + if (lstat(entry.c_str(), &st)) + throw SysError("getting attributes of path '%1%'", entry); + if (S_ISREG(st.st_mode)) { + createRegularFile(destination + "/" + i.name, st.st_mode & S_IXUSR); + copyFile(entry); + } else if (S_ISDIR(st.st_mode)) + copyDirectory(entry, destination + "/" + i.name); + else + throw Error("Unknown file: %s", entry); + } +} + + +} diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh new file mode 100644 index 00000000000..dfb9c67c193 --- /dev/null +++ b/src/libutil/fs-sink.hh @@ -0,0 +1,48 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + +namespace nix { + +/** + * \todo Fix this API, it sucks. + */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path, bool executable = false) { }; + virtual void closeRegularFile() { }; + virtual void isExecutable() { }; + virtual void preallocateContents(uint64_t size) { }; + virtual void receiveContents(std::string_view data) { }; + + virtual void createSymlink(const Path & path, const std::string & target) { }; + + virtual void copyFile(const Path & source) { }; + virtual void copyDirectory(const Path & source, const Path & destination) { }; +}; + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path, bool executable = false) override; + void closeRegularFile() override; + void isExecutable() override; + void preallocateContents(uint64_t size) override; + void receiveContents(std::string_view data) override; + + void createSymlink(const Path & path, const std::string & target) override; + + void copyFile(const Path & source) override; + void copyDirectory(const Path & source, const Path & destination) override; +}; + + +} diff --git a/src/libutil/git.cc b/src/libutil/git.cc index f35c2fdb75c..9bfdd48ac5e 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -1,8 +1,238 @@ -#include "git.hh" +#include +#include +#include +#include #include +#include // for strcasecmp + +#include +#include +#include +#include +#include + +#include "util.hh" +#include "config.hh" +#include "hash.hh" + +#include "git.hh" +#include "serialise.hh" + +using namespace std::string_literals; + namespace nix { + +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir); + +// Converts a Path to a ParseSink +void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir) { + RestoreSink sink; + sink.dstPath = path; + parseGit(sink, source, realStoreDir, storeDir); +} + +void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir) +{ + parse(sink, source, "", realStoreDir, storeDir); +} + +static std::string getStringUntil(Source & source, char byte) +{ + std::string s; + char n[1]; + source(std::string_view { n, 1 }); + while (*n != byte) { + s += *n; + source(std::string_view { n, 1 }); + } + return s; +} + +static std::string getString(Source & source, int n) +{ + std::string v; + v.resize(n); + source(v); + return v; +} + +// Unfortunately, no access to libstore headers here. +static std::string getStoreEntry(const Path & storeDir, Hash hash, std::string name) +{ + Hash hash1 = hashString(htSHA256, "fixed:out:git:" + hash.to_string(Base::Base16, true) + ":"); + Hash hash2 = hashString(htSHA256, "output:out:" + hash1.to_string(Base::Base16, true) + ":" + storeDir + ":" + name); + Hash hash3 = compressHash(hash2, 20); + + return hash3.to_string(Base::Base32, false) + "-" + name; +} + +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir) +{ + auto type = getString(source, 5); + + if (type == "blob ") { + sink.createRegularFile(path); + + unsigned long long size = std::stoi(getStringUntil(source, 0)); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::string buf; + buf.reserve(65536); + + while (left) { + checkInterrupt(); + buf.resize(std::min((unsigned long long)buf.capacity(), left)); + source(buf); + sink.receiveContents(buf); + left -= buf.size(); + } + } else if (type == "tree ") { + unsigned long long size = std::stoi(getStringUntil(source, 0)); + unsigned long long left = size; + + sink.createDirectory(path); + + while (left) { + std::string perms = getStringUntil(source, ' '); + left -= perms.size(); + left -= 1; + + int perm = std::stoi(perms); + if (perm != 100644 && perm != 100755 && perm != 644 && perm != 755 && perm != 40000) + throw Error("Unknown Git permission: %d", perm); + + std::string name = getStringUntil(source, 0); + left -= name.size(); + left -= 1; + + std::string hashs = getString(source, 20); + left -= 20; + + Hash hash(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); + + std::string entryName = getStoreEntry(storeDir, hash, "git"); + Path entry = absPath(realStoreDir + "/" + entryName); + + struct stat st; + if (lstat(entry.c_str(), &st)) + throw SysError("getting attributes of path '%1%'", entry); + + if (S_ISREG(st.st_mode)) { + if (perm == 40000) + throw SysError("file is a file but expected to be a directory '%1%'", entry); + + sink.createRegularFile(path + "/" + name, perm == 100755 || perm == 755); + + sink.copyFile(entry); + } else if (S_ISDIR(st.st_mode)) { + if (perm != 40000) + throw SysError("file is a directory but expected to be a file '%1%'", entry); + + sink.copyDirectory(realStoreDir + "/" + entryName, path + "/" + name); + } else throw Error("file '%1%' has an unsupported type", entry); + } + } else throw Error("input doesn't look like a Git object"); +} + +// TODO stream file into sink, rather than reading into vector +GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink) +{ + auto s = fmt("blob %d\0%s"s, std::to_string(st.st_size), readFile(path)); + + std::string v; + std::copy(s.begin(), s.end(), std::back_inserter(v)); + sink(v); + return st.st_mode & S_IXUSR + ? GitMode::Executable + : GitMode::Regular; +} + +GitMode dumpGitTree(const GitTree & entries, Sink & sink) +{ + std::vector v1; + + for (auto & i : entries) { + unsigned int mode; + switch (i.second.first) { + case GitMode::Directory: mode = 40000; break; + case GitMode::Executable: mode = 100755; break; + case GitMode::Regular: mode = 100644; break; + } + auto name = i.first; + if (i.second.first == GitMode::Directory) + name.pop_back(); + auto s1 = fmt("%d %s", mode, name); + std::copy(s1.begin(), s1.end(), std::back_inserter(v1)); + v1.push_back(0); + std::copy(i.second.second.hash, i.second.second.hash + 20, std::back_inserter(v1)); + } + + std::string v2; + auto s2 = fmt("tree %d"s, v1.size()); + std::copy(s2.begin(), s2.end(), std::back_inserter(v2)); + v2.push_back(0); + std::copy(v1.begin(), v1.end(), std::back_inserter(v2)); + + sink(v2); + + return GitMode::Directory; +} + +static std::pair dumpGitHashInternal(HashType ht, const Path & path, PathFilter & filter); + +static GitMode dumpGitInternal(HashType ht, const Path & path, Sink & sink, PathFilter & filter) +{ + struct stat st; + GitMode perm; + if (lstat(path.c_str(), &st)) + throw SysError("getting attributes of path '%1%'", path); + + if (S_ISREG(st.st_mode)) + perm = dumpGitBlob(path, st, sink); + else if (S_ISDIR(st.st_mode)) { + GitTree entries; + for (auto & i : readDirectory(path)) + if (filter(path + "/" + i.name)) { + auto result = dumpGitHashInternal(ht, path + "/" + i.name, filter); + + // correctly observe git order, see + // https://github.com/mirage/irmin/issues/352 + auto name = i.name; + if (result.first == GitMode::Directory) + name += "/"; + + entries.insert_or_assign(name, result); + } + perm = dumpGitTree(entries, sink); + } else throw Error("file '%1%' has an unsupported type", path); + + return perm; +} + + +static std::pair dumpGitHashInternal(HashType ht, const Path & path, PathFilter & filter) +{ + auto hashSink = new HashSink(ht); + auto perm = dumpGitInternal(ht, path, *hashSink, filter); + auto hash = hashSink->finish().first; + return std::pair { perm, hash }; +} + +Hash dumpGitHash(HashType ht, const Path & path, PathFilter & filter) +{ + return dumpGitHashInternal(ht, path, filter).second; +} + +void dumpGit(HashType ht, const Path & path, Sink & sink, PathFilter & filter) +{ + dumpGitInternal(ht, path, sink, filter); +} + namespace git { std::optional parseLsRemoteLine(std::string_view line) @@ -22,4 +252,5 @@ std::optional parseLsRemoteLine(std::string_view line) } } + } diff --git a/src/libutil/git.hh b/src/libutil/git.hh index bf2b9a2869a..e39dfae163b 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -5,8 +5,36 @@ #include #include +#include "types.hh" +#include "serialise.hh" +#include "hash.hh" +#include "fs-sink.hh" + namespace nix { +enum struct GitMode { + Directory, + Executable, + Regular, +}; + +void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir); + +void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir); + +// Dumps a single file to a sink +GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink); + +typedef std::map> GitTree; + +// Dumps a representation of a git tree to a sink +GitMode dumpGitTree(const GitTree & entries, Sink & sink); + +// Recursively dumps path, hashing as we go +Hash dumpGitHash(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); + +void dumpGit(HashType ht, const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); + namespace git { /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2c36d9d9498..f04c6b9041d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -8,6 +8,7 @@ #include "args.hh" #include "hash.hh" #include "archive.hh" +#include "git.hh" #include "split.hh" #include "util.hh" @@ -375,6 +376,13 @@ HashResult hashPath( return sink.finish(); } +HashResult hashGit( + HashType ht, const Path & path, PathFilter & filter) +{ + HashSink sink(ht); + dumpGit(ht, path, sink, filter); + return sink.finish(); +} Hash compressHash(const Hash & hash, unsigned int newSize) { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ae3ee40f4e3..0d4abd72b8f 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -145,18 +145,28 @@ std::string printHash16or32(const Hash & hash); Hash hashString(HashType ht, std::string_view s); /** - * Compute the hash of the given file. + * Compute the hash of the given file, hashing its contents directly. + * + * (Metadata, such as the executable permission bit, is ignored.) */ Hash hashFile(HashType ht, const Path & path); /** - * Compute the hash of the given path. The hash is defined as - * (essentially) hashString(ht, dumpPath(path)). + * Compute the hash of the given path, serializing as a Nix Archive and + * then hashing that. + * + * The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); +/** + * Compute the git blob/tree hash of the given path. + */ +HashResult hashGit(HashType ht, const Path & path, + PathFilter & filter = defaultPathFilter); + /** * Compress a hash to the specified number of bytes by cyclically * XORing bytes together. diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh new file mode 100644 index 00000000000..ac4a9994e5a --- /dev/null +++ b/src/libutil/reference-set.hh @@ -0,0 +1,68 @@ +#pragma once + +#include "comparator.hh" + +#include + +namespace nix { + +template +struct References +{ + std::set others; + bool self = false; + + bool empty() const; + size_t size() const; + + /* Functions to view references + self as one set, mainly for + compatibility's sake. */ + std::set possiblyToSelf(const Ref & self) const; + void insertPossiblyToSelf(const Ref & self, Ref && ref); + void setPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(References, me->others, me->self); +}; + +template +bool References::empty() const +{ + return !self && others.empty(); +} + +template +size_t References::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +template +std::set References::possiblyToSelf(const Ref & selfRef) const +{ + std::set refs { others }; + if (self) + refs.insert(selfRef); + return refs; +} + +template +void References::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) +{ + if (ref == selfRef) + self = true; + else + others.insert(std::move(ref)); +} + +template +void References::setPossiblyToSelf(const Ref & selfRef, std::set && refs) +{ + if (refs.count(selfRef)) { + self = true; + refs.erase(selfRef); + } + + others = refs; +} + +} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 3d5121a19fa..e5205ce79f0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -74,6 +74,10 @@ void Source::operator () (char * data, size_t len) } } +void Source::operator () (std::string_view data) +{ + (*this)((char *)data.data(), data.size()); +} void Source::drainInto(Sink & sink) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 333c254ea8e..1a714e87cdd 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -72,6 +72,7 @@ struct Source * an error if it is not going to be available. */ void operator () (char * data, size_t len); + void operator () (std::string_view data); /** * Store up to ‘len’ in the buffer pointed to by ‘data’, and @@ -232,6 +233,7 @@ struct TeeSource : Source } }; + /** * A reader that consumes the original Source until 'size'. */ diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 95f4014411f..8eab069c9d5 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -113,7 +113,8 @@ static void update(const StringSet & channelNames) // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); - auto filename = store->toRealPath(result.storePath); + auto filename = store->toRealPath( + store->makeFixedOutputPathFromCA(result.storePath)); url = result.effectiveUrl; bool unpacked = false; @@ -124,12 +125,21 @@ static void update(const StringSet & channelNames) } if (!unpacked) { + StorePathDescriptor storePathDesc { + .name = "t e m p", + .info = FixedOutputInfo { + .method = FileIngestionMethod::Flat, + .hash = Hash(htSHA256), + .references = {}, + }, + }; // Download the channel tarball. try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); + storePathDesc = fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath; } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); + storePathDesc = fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath; } + filename = store->toRealPath(store->makeFixedOutputPathFromCA(storePathDesc)); } // Regardless of where it came from, add the expression representing this channel to accumulated expression exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }"); diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb31..36d774dca76 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -56,7 +56,7 @@ void printDotGraph(ref store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 4395576589c..01a3a605fb3 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->references) { + for (auto & p : info->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 96c3f7d7e03..e41652f7639 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -265,7 +265,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->references); + auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -347,7 +347,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) + for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) paths.insert(p); } else if (query == qReferrers) { @@ -368,7 +368,8 @@ static void opQuery(Strings opFlags, Strings opArgs) case qDeriver: for (auto & i : opArgs) { - auto info = store->queryPathInfo(store->followLinksToStorePath(i)); + auto path = store->followLinksToStorePath(i); + auto info = store->queryPathInfo(path); cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver"); } break; @@ -888,7 +889,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - WorkerProto::write(*store, wconn, info->references); + WorkerProto::write(*store, wconn, info->referencesPossiblyToSelf()); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -903,9 +904,11 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case ServeProto::Command::DumpStorePath: - store->narFromPath(store->parseStorePath(readString(in)), out); + case ServeProto::Command::DumpStorePath: { + auto path = store->parseStorePath(readString(in)); + store->narFromPath(path, out); break; + } case ServeProto::Command::ImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); @@ -988,7 +991,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, rconn); + info.setReferencesPossiblyToSelf(WorkerProto::Serialise::read(*store, rconn)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 39e5cc99dd2..b19b67bde55 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "store-api.hh" #include "archive.hh" +#include "git.hh" using namespace nix; @@ -34,20 +35,33 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, sink.s); - Hash hash = narHash; - if (ingestionMethod == FileIngestionMethod::Flat) { + Hash hash { htSHA256 }; // throwaway def to appease C++ + switch (ingestionMethod) { + case FileIngestionMethod::Recursive: { + hash = narHash; + break; + } + case FileIngestionMethod::Flat: { HashSink hsink(htSHA256); readFile(path, hsink); hash = hsink.finish().first; + break; + } + case FileIngestionMethod::Git: { + hash = dumpGitHash(htSHA1, path); + break; + } } ValidPathInfo info { *store, - std::move(*namePart), - FixedOutputInfo { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - .references = {}, + StorePathDescriptor { + .name = std::move(*namePart), + .info = FixedOutputInfo { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + .references = {}, + }, }, narHash, }; @@ -102,5 +116,26 @@ struct CmdAddPath : CmdAddToStore } }; +struct CmdAddGit : CmdAddToStore +{ + CmdAddGit() + { + ingestionMethod = FileIngestionMethod::Git; + } + + std::string description() override + { + return "add a path to the Nix store"; + } + + std::string doc() override + { + return + #include "add-path.md" + ; + } +}; + static auto rCmdAddFile = registerCommand2({"store", "add-file"}); static auto rCmdAddPath = registerCommand2({"store", "add-path"}); +static auto rCmdAddGit = registerCommand2({"store", "add-git"}); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 87dd4da1bec..3b0e8c934d6 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -186,7 +186,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["path"] = store->printStorePath( + store->makeFixedOutputPathFromCA(flake.sourceInfo->storePath)); j["locks"] = lockedFlake.lockFile.toJSON(); logger->cout("%s", j.dump()); } else { @@ -202,7 +203,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON *flake.description); logger->cout( ANSI_BOLD "Path:" ANSI_NORMAL " %s", - store->printStorePath(flake.sourceInfo->storePath)); + store->printStorePath( + store->makeFixedOutputPathFromCA(flake.sourceInfo->storePath))); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -972,7 +974,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - sources.insert(flake.flake.sourceInfo->storePath); + sources.insert( + store->makeFixedOutputPathFromCA(flake.flake.sourceInfo->storePath)); // FIXME: use graph output, handle cycles. std::function traverse; @@ -981,10 +984,11 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun nlohmann::json jsonObj2 = json ? json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { - auto storePath = + auto storePathDesc = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + auto storePath = store->makeFixedOutputPathFromCA(storePathDesc); if (json) { auto& jsonObj3 = jsonObj2[inputName]; jsonObj3["path"] = store->printStorePath(storePath); @@ -1001,8 +1005,16 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (json) { nlohmann::json jsonRoot = { - {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, - {"inputs", traverse(*flake.lockFile.root)}, + { + "path", + store->printStorePath( + store->makeFixedOutputPathFromCA( + flake.flake.sourceInfo->storePath)), + }, + { + "inputs", + traverse(*flake.lockFile.root), + }, }; logger->cout("%s", jsonRoot); } else { @@ -1338,15 +1350,16 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON auto [tree, lockedRef] = resolvedRef.fetchTree(store); auto hash = store->queryPathInfo(tree.storePath)->narHash; + auto actualStorePath = store->bakeCaIfNeeded((const OwnedStorePathOrDesc &)tree.storePath); if (json) { auto res = nlohmann::json::object(); - res["storePath"] = store->printStorePath(tree.storePath); + res["storePath"] = store->printStorePath(actualStorePath); res["hash"] = hash.to_string(SRI, true); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), - store->printStorePath(tree.storePath), + store->printStorePath(actualStorePath), hash.to_string(SRI, true)); } } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9feca934557..368227ab628 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -5,6 +5,7 @@ #include "shared.hh" #include "references.hh" #include "archive.hh" +#include "git.hh" using namespace nix; @@ -65,9 +66,11 @@ struct CmdHashBase : Command { switch (mode) { case FileIngestionMethod::Flat: - return "print cryptographic hash of a regular file"; + return "print cryptographic hash of a regular file"; case FileIngestionMethod::Recursive: return "print cryptographic hash of the NAR serialisation of a path"; + case FileIngestionMethod::Git: + return "print cryptographic hash of the Git serialisation of a path"; default: assert(false); }; @@ -84,15 +87,21 @@ struct CmdHashBase : Command hashSink = std::make_unique(ht); switch (mode) { - case FileIngestionMethod::Flat: + case FileIngestionMethod::Flat: { readFile(path, *hashSink); break; - case FileIngestionMethod::Recursive: + } + case FileIngestionMethod::Recursive: { dumpPath(path, *hashSink); break; } + case FileIngestionMethod::Git: + dumpGit(ht, path, *hashSink); + break; + } + + auto h = hashSink->finish().first; - Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); logger->cout(h.to_string(base, base == SRI)); } @@ -133,6 +142,7 @@ struct CmdHash : NixMultiCommand : MultiCommand({ {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, + {"git", []() { return make_ref(FileIngestionMethod::Git); }}, {"to-base16", []() { return make_ref(Base16); }}, {"to-base32", []() { return make_ref(Base32); }}, {"to-base64", []() { return make_ref(Base64); }}, diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 476ddcd609b..65df6fdc863 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -220,14 +220,16 @@ struct ProfileManifest ValidPathInfo info { *store, - "profile", - FixedOutputInfo { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - .references = { - .others = std::move(references), - // profiles never refer to themselves - .self = false, + StorePathDescriptor { + "profile", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + .references = { + .others = std::move(references), + // profiles never refer to themselves + .self = false, + }, }, }, narHash, diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index d238456db5b..c3010711856 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -132,7 +132,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); - if (!store->isValidPath(store->parseStorePath(userEnv))) + auto path = store->parseStorePath(userEnv); + if (!store->isValidPath(path)) throw Error("directory '%s' is not in the Nix store", userEnv); return profileDir; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 592de773ce9..7818fda9ff6 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references, + .refs = store->queryPathInfo(path)->references.others, .dist = path == dependencyPath ? 0 : inf }); diff --git a/tests/git.sh b/tests/git.sh new file mode 100644 index 00000000000..983033e6a28 --- /dev/null +++ b/tests/git.sh @@ -0,0 +1,83 @@ +source common.sh + +clearStore +clearCache + +enableFeatures "git-hashing" + +try () { + hash=$(nix hash git --base16 --type sha1 $TEST_ROOT/hash-path) + if test "$hash" != "$1"; then + echo "git hash, expected $1, got $hash" + exit 1 + fi +} + +rm -rf $TEST_ROOT/hash-path +mkdir $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path/hello + +try "117c62a8c5e01758bd284126a6af69deab9dbbe2" + +rm -rf $TEST_ROOT/dummy1 +echo Hello World! > $TEST_ROOT/dummy1 +path1=$(nix store add-git $TEST_ROOT/dummy1) +hash1=$(nix-store -q --hash $path1) +test "$hash1" = "sha256:1brffhvj2c0z6x8qismd43m0iy8dsgfmy10bgg9w11szway2wp9v" + +rm -rf $TEST_ROOT/dummy2 +mkdir -p $TEST_ROOT/dummy2 +echo Hello World! > $TEST_ROOT/dummy2/hello +path2=$(nix store add-git $TEST_ROOT/dummy2) +hash2=$(nix-store -q --hash $path2) +test "$hash2" = "sha256:1vhv7zxam7x277q0y0jcypm7hwhccbzss81vkdgf0ww5sm2am4y0" + +rm -rf $TEST_ROOT/dummy3 +mkdir -p $TEST_ROOT/dummy3 +mkdir -p $TEST_ROOT/dummy3/hello +echo Hello World! > $TEST_ROOT/dummy3/hello/hello +path3=$(nix store add-git $TEST_ROOT/dummy3) +hash3=$(nix-store -q --hash $path3) +test "$hash3" = "sha256:1i2x80840igikhbyy7nqf08ymx3a6n83x1fzyrxvddf0sdl5nqvp" + +if [[ -n $(type -p git) ]]; then + repo=$TEST_ROOT/git + + rm -rf $repo $TEST_HOME/.cache/nix + + git init $repo + git -C $repo config user.email "foobar@example.com" + git -C $repo config user.name "Foobar" + + echo utrecht > $repo/hello + touch $repo/.gitignore + git -C $repo add hello .gitignore + git -C $repo commit -m 'Bla1' + + echo world > $repo/hello + git -C $repo commit -m 'Bla2' -a + + treeHash=$(git -C $repo rev-parse HEAD:) + + # Fetch the default branch. + path=$(nix eval --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$repo; treeHash = \"$treeHash\"; }).outPath") + [[ $(cat $path/hello) = world ]] + + # Submodules cause error. + (! nix eval --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$repo; treeHash = \"$treeHash\"; submodules = true; }).outPath") + + # Check that we can substitute it from other places. + nix copy --to file://$cacheDir $path + nix-store --delete $path + path2=$(nix eval --raw --expr "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$treeHash\"; }).outPath" --substituters file://$cacheDir --option substitute true) + [ $path2 = $path ] + + # HEAD should be the same path and tree hash as tree + nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = file://$repo; ref = \"HEAD\"; gitIngestion = true; })" + treeHash2=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$repo; ref = \"HEAD\"; gitIngestion = true; }).treeHash") + [ $treeHash = $treeHash2 ] + path3=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$repo; ref = \"HEAD\"; gitIngestion = true; }).outPath") + [ $path3 = $path ] +else + echo "Git not installed; skipping Git tests" +fi diff --git a/tests/local.mk b/tests/local.mk index 4edf31303a9..f49f1a3ad26 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -23,6 +23,7 @@ nix_tests = \ experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ + git.sh \ user-envs.sh \ user-envs-migration.sh \ binary-cache.sh \