From 81ebc38f8050eae6138bfac5a81502aa629d307b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 4 Nov 2023 16:25:41 -0400 Subject: [PATCH] Organize content addressing, use `SourceAccessor` with `Store::addToStore` Co-authored-by: Robert Hensing --- src/libcmd/installable-value.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 2 +- src/libfetchers/cache.cc | 16 ++-- src/libfetchers/cache.hh | 6 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/github.cc | 4 +- src/libfetchers/input-accessor.cc | 39 ++++++--- src/libfetchers/input-accessor.hh | 8 +- src/libfetchers/mercurial.cc | 21 +++-- src/libfetchers/tarball.cc | 14 +-- src/libstore/binary-cache-store.cc | 50 ++++++----- src/libstore/binary-cache-store.hh | 18 ++-- src/libstore/build/local-derivation-goal.cc | 80 ++++++++--------- src/libstore/build/worker.cc | 4 +- src/libstore/content-address.cc | 22 +++-- src/libstore/content-address.hh | 36 ++++---- src/libstore/daemon.cc | 19 +---- src/libstore/legacy-ssh-store.cc | 9 +- src/libstore/local-store.cc | 95 ++++++++------------- src/libstore/local-store.hh | 22 ++--- src/libstore/optimise-store.cc | 15 +++- src/libstore/remote-store.cc | 9 +- src/libstore/remote-store.hh | 11 ++- src/libstore/store-api.cc | 78 ++++++++++------- src/libstore/store-api.hh | 42 +++++---- src/libutil/file-content-address.cc | 49 +++++++++++ src/libutil/file-content-address.hh | 56 ++++++++++++ src/libutil/hash.cc | 9 -- src/libutil/hash.hh | 7 +- src/nix-store/nix-store.cc | 20 ++++- src/nix/add-to-store.cc | 48 ++++------- src/nix/hash.cc | 11 +-- src/nix/prefetch.cc | 7 +- 34 files changed, 475 insertions(+), 358 deletions(-) create mode 100644 src/libutil/file-content-address.cc create mode 100644 src/libutil/file-content-address.hh diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 08ad35105bc..bdc34bbe353 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -44,7 +44,7 @@ ref InstallableValue::require(ref installable) std::optional InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) { if (v.type() == nPath) { - auto storePath = v.path().fetchToStore(state->store); + auto storePath = v.path().fetchToStore(*state->store); return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index bf6b6f8c143..a46f29e28b9 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2345,7 +2345,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); + auto dstPath = path.fetchToStore(*store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7c056141372..5e74a27819a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2241,7 +2241,7 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair); + auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(*state.store, name, method, &filter, state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 63b05bdab19..e071b4717b4 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -106,7 +106,7 @@ struct CacheImpl : Cache } void add( - ref store, + Store & store, const Attrs & inAttrs, const Attrs & infoAttrs, const StorePath & storePath, @@ -115,13 +115,13 @@ struct CacheImpl : Cache _state.lock()->add.use() (attrsToJSON(inAttrs).dump()) (attrsToJSON(infoAttrs).dump()) - (store->printStorePath(storePath)) + (store.printStorePath(storePath)) (locked) (time(0)).exec(); } std::optional> lookup( - ref store, + Store & store, const Attrs & inAttrs) override { if (auto res = lookupExpired(store, inAttrs)) { @@ -134,7 +134,7 @@ struct CacheImpl : Cache } std::optional lookupExpired( - ref store, + Store & store, const Attrs & inAttrs) override { auto state(_state.lock()); @@ -148,19 +148,19 @@ struct CacheImpl : Cache } auto infoJSON = stmt.getStr(0); - auto storePath = store->parseStorePath(stmt.getStr(1)); + auto storePath = store.parseStorePath(stmt.getStr(1)); auto locked = stmt.getInt(2) != 0; auto timestamp = stmt.getInt(3); - store->addTempRoot(storePath); - if (!store->isValidPath(storePath)) { + store.addTempRoot(storePath); + if (!store.isValidPath(storePath)) { // FIXME: we could try to substitute 'storePath'. debug("ignoring disappeared cache entry '%s'", inAttrsJSON); return {}; } debug("using cache entry '%s' -> '%s', '%s'", - inAttrsJSON, infoJSON, store->printStorePath(storePath)); + inAttrsJSON, infoJSON, store.printStorePath(storePath)); return Result { .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index f70589267f7..791d77025aa 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -50,14 +50,14 @@ struct Cache /* Old cache for things that have a store path. */ virtual void add( - ref store, + Store & store, const Attrs & inAttrs, const Attrs & infoAttrs, const StorePath & storePath, bool locked) = 0; virtual std::optional> lookup( - ref store, + Store & store, const Attrs & inAttrs) = 0; struct Result @@ -68,7 +68,7 @@ struct Cache }; virtual std::optional lookupExpired( - ref store, + Store & store, const Attrs & inAttrs) = 0; }; diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 60208619e87..070526849b1 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -374,7 +374,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const std::pair InputScheme::fetch(ref store, const Input & input) { auto [accessor, input2] = getAccessor(store, input); - auto storePath = accessor->root().fetchToStore(store, input2.getName()); + auto storePath = accessor->root().fetchToStore(*store, input2.getName()); return {storePath, input2}; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 661ad48844a..90249ef5e6f 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -201,7 +201,7 @@ struct GitArchiveInputScheme : InputScheme {"rev", rev->gitRev()}, }); - if (auto res = getCache()->lookup(store, lockedAttrs)) { + if (auto res = getCache()->lookup(*store, lockedAttrs)) { input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); return {std::move(res->second), input}; } @@ -213,7 +213,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); getCache()->add( - store, + *store, lockedAttrs, { {"rev", rev->gitRev()}, diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 85dc4609fe5..6c468a42923 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -5,10 +5,10 @@ namespace nix { StorePath InputAccessor::fetchToStore( - ref store, + Store & store, const CanonPath & path, std::string_view name, - FileIngestionMethod method, + ContentAddressMethod method, PathFilter * filter, RepairFlag repair) { @@ -20,10 +20,24 @@ StorePath InputAccessor::fetchToStore( if (!filter && fingerprint) { cacheKey = fetchers::Attrs{ {"_what", "fetchToStore"}, - {"store", store->storeDir}, + {"store", store.storeDir}, {"name", std::string(name)}, {"fingerprint", *fingerprint}, - {"method", (uint8_t) method}, + { + "method", + std::visit(overloaded { + [](const TextIngestionMethod &) { + return "text"; + }, + [](const FileIngestionMethod & fim) { + switch (fim) { + case FileIngestionMethod::Flat: return "flat"; + case FileIngestionMethod::Recursive: return "nar"; + default: assert(false); + } + }, + }, method.raw), + }, {"path", path.abs()} }; if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { @@ -35,17 +49,14 @@ StorePath InputAccessor::fetchToStore( Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); - auto source = sinkToSource([&](Sink & sink) { - if (method == FileIngestionMethod::Recursive) - dumpPath(path, sink, filter ? *filter : defaultPathFilter); - else - readFile(path, sink); - }); + auto filter2 = filter ? *filter : defaultPathFilter; auto storePath = settings.readOnlyMode - ? store->computeStorePathFromDump(*source, name, method, htSHA256).first - : store->addToStoreFromDump(*source, name, method, htSHA256, repair); + ? store.computeStorePath( + name, *this, path, method, htSHA256, {}, filter2).first + : store.addToStore( + name, *this, path, method, htSHA256, {}, filter2, repair); if (cacheKey) fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); @@ -65,9 +76,9 @@ std::ostream & operator << (std::ostream & str, const SourcePath & path) } StorePath SourcePath::fetchToStore( - ref store, + Store & store, std::string_view name, - FileIngestionMethod method, + ContentAddressMethod method, PathFilter * filter, RepairFlag repair) const { diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 26d17f06428..c7c7650376b 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -30,10 +30,10 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this store, + Store & store, const CanonPath & path, std::string_view name = "source", - FileIngestionMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair); @@ -113,9 +113,9 @@ struct SourcePath * Copy this `SourcePath` to the Nix store. */ StorePath fetchToStore( - ref store, + Store & store, std::string_view name = "source", - FileIngestionMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair) const; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index aa991a75d6f..c5721f3fb27 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -6,6 +6,7 @@ #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" +#include "posix-source-accessor.hh" #include "fetch-settings.hh" @@ -210,7 +211,12 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + PosixSourceAccessor accessor; + auto storePath = store->addToStore( + input.getName(), + accessor, CanonPath { actualPath }, + FileIngestionMethod::Recursive, htSHA256, {}, + filter); return {std::move(storePath), input}; } @@ -246,7 +252,7 @@ struct MercurialInputScheme : InputScheme }; if (input.getRev()) { - if (auto res = getCache()->lookup(store, getLockedAttrs())) + if (auto res = getCache()->lookup(*store, getLockedAttrs())) return makeResult(res->first, std::move(res->second)); } @@ -259,7 +265,7 @@ struct MercurialInputScheme : InputScheme {"ref", *input.getRef()}, }); - if (auto res = getCache()->lookup(store, unlockedAttrs)) { + if (auto res = getCache()->lookup(*store, unlockedAttrs)) { auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); @@ -305,7 +311,7 @@ struct MercurialInputScheme : InputScheme auto revCount = std::stoull(tokens[1]); input.attrs.insert_or_assign("ref", tokens[2]); - if (auto res = getCache()->lookup(store, getLockedAttrs())) + if (auto res = getCache()->lookup(*store, getLockedAttrs())) return makeResult(res->first, std::move(res->second)); Path tmpDir = createTempDir(); @@ -315,7 +321,8 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir + "/.hg_archival.txt"); - auto storePath = store->addToStore(name, tmpDir); + PosixSourceAccessor accessor; + auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir }); Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, @@ -324,14 +331,14 @@ struct MercurialInputScheme : InputScheme if (!_input.getRev()) getCache()->add( - store, + *store, unlockedAttrs, infoAttrs, storePath, false); getCache()->add( - store, + *store, getLockedAttrs(), infoAttrs, storePath, diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 0062878a9d4..7031589bcef 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -8,6 +8,7 @@ #include "tarfile.hh" #include "types.hh" #include "split.hh" +#include "posix-source-accessor.hh" namespace nix::fetchers { @@ -26,7 +27,7 @@ DownloadFileResult downloadFile( {"name", name}, }); - auto cached = getCache()->lookupExpired(store, inAttrs); + auto cached = getCache()->lookupExpired(*store, inAttrs); auto useCached = [&]() -> DownloadFileResult { @@ -91,7 +92,7 @@ DownloadFileResult downloadFile( } getCache()->add( - store, + *store, inAttrs, infoAttrs, *storePath, @@ -99,7 +100,7 @@ DownloadFileResult downloadFile( if (url != res.effectiveUri) getCache()->add( - store, + *store, { {"type", "file"}, {"url", res.effectiveUri}, @@ -130,7 +131,7 @@ DownloadTarballResult downloadTarball( {"name", name}, }); - auto cached = getCache()->lookupExpired(store, inAttrs); + auto cached = getCache()->lookupExpired(*store, inAttrs); if (cached && !cached->expired) return { @@ -156,7 +157,8 @@ DownloadTarballResult downloadTarball( throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + PosixSourceAccessor accessor; + unpackedStorePath = store->addToStore(name, accessor, CanonPath { topDir }, FileIngestionMethod::Recursive, htSHA256, {}, defaultPathFilter, NoRepair); } Attrs infoAttrs({ @@ -168,7 +170,7 @@ DownloadTarballResult downloadTarball( infoAttrs.emplace("immutableUrl", *res.immutableUrl); getCache()->add( - store, + *store, inAttrs, infoAttrs, *unpackedStorePath, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ae483c95efc..114c7b07844 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -300,8 +300,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource }}); } -StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) +StorePath BinaryCacheStore::addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashType hashAlgo, + const StorePathSet & references, + RepairFlag repair) { if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) unsupported("addToStoreFromDump"); @@ -309,15 +314,14 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n ValidPathInfo info { *this, name, - FixedOutputInfo { - .method = method, - .hash = nar.first, - .references = { + ContentAddressWithReferences::fromParts( + method, + nar.first, + { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, - }, - }, + }), nar.first, }; info.narSize = nar.second; @@ -400,41 +404,35 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, StorePath BinaryCacheStore::addToStore( std::string_view name, - const Path & srcPath, - FileIngestionMethod method, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method, HashType hashAlgo, + const StorePathSet & references, PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + RepairFlag repair) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default implementation of this method in terms of addToStoreFromDump. */ - HashSink sink { hashAlgo }; - if (method == FileIngestionMethod::Recursive) { - dumpPath(srcPath, sink, filter); - } else { - readFile(srcPath, sink); - } - auto h = sink.finish().first; + auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first; auto source = sinkToSource([&](Sink & sink) { - dumpPath(srcPath, sink, filter); + accessor.dumpPath(path, sink, filter); }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, name, - FixedOutputInfo { - .method = method, - .hash = h, - .references = { + ContentAddressWithReferences::fromParts( + method, + h, + { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, - }, - }, + }), nar.first, }; info.narSize = nar.second; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index cea2a571f13..1d3b934aba4 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -123,17 +123,23 @@ public: void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashType hashAlgo, + const StorePathSet & references, + RepairFlag repair) override; StorePath addToStore( std::string_view name, - const Path & srcPath, - FileIngestionMethod method, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, HashType hashAlgo, + const StorePathSet & references, PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override; + RepairFlag repair) override; StorePath addTextToStore( std::string_view name, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 198402ff7c8..2a7086f8f5f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -20,6 +20,7 @@ #include "child.hh" #include "unix-domain-socket.hh" #include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" #include #include @@ -1291,12 +1292,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In StorePath addToStore( std::string_view name, - const Path & srcPath, - FileIngestionMethod method, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, HashType hashAlgo, + const StorePathSet & references, PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + RepairFlag repair) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1320,12 +1322,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileIngestionMethod method, + ContentAddressMethod method, HashType hashAlgo, - RepairFlag repair, - const StorePathSet & references) override + const StorePathSet & references, + RepairFlag repair) override { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair); goal.addDependency(path); return path; } @@ -2453,8 +2455,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() throw BuildError( "output path %1% without valid stats info", actualPath); - if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } || - outputHash.method == ContentAddressMethod { TextIngestionMethod {} }) + if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) @@ -2466,38 +2467,23 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - std::visit(overloaded { - [&](const TextIngestionMethod &) { - readFile(actualPath, caSink); - }, - [&](const FileIngestionMethod & m2) { - switch (m2) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - }, - }, outputHash.method.raw); - auto got = caSink.finish().first; - - auto optCA = ContentAddressWithReferences::fromPartsOpt( - outputHash.method, - std::move(got), - rewriteRefs()); - if (!optCA) { - // TODO track distinct failure modes separately (at the time of - // writing there is just one but `nullopt` is unclear) so this - // message can't get out of sync. - throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing"); - } + auto got = ({ + HashModuloSink caSink { outputHash.hashType, oldHashPart }; + PosixSourceAccessor accessor; + dumpPath( + accessor, CanonPath { actualPath }, + caSink, + outputHash.method.getFileIngestionMethod()); + caSink.finish().first; + }); + ValidPathInfo newInfo0 { worker.store, outputPathName(drv->name, outputName), - std::move(*optCA), + ContentAddressWithReferences::fromParts( + outputHash.method, + std::move(got), + rewriteRefs()), Hash::dummy, }; if (*scratchPath != newInfo0.path) { @@ -2511,9 +2497,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string(newInfo0.path.hashPart())}}); } - HashResult narHashAndSize = hashPath(htSHA256, actualPath); - newInfo0.narHash = narHashAndSize.first; - newInfo0.narSize = narHashAndSize.second; + { + PosixSourceAccessor accessor; + HashResult narHashAndSize = hashPath( + accessor, CanonPath { actualPath }, + FileIngestionMethod::Recursive, htSHA256); + newInfo0.narHash = narHashAndSize.first; + newInfo0.narSize = narHashAndSize.second; + } assert(newInfo0.ca); return newInfo0; @@ -2531,7 +2522,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); rewriteOutput(outputRewrites); - auto narHashAndSize = hashPath(htSHA256, actualPath); + PosixSourceAccessor accessor; + HashResult narHashAndSize = hashPath( + accessor, CanonPath { actualPath }, + FileIngestionMethod::Recursive, htSHA256); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 01f52e7aba7..0b7717c3d01 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -519,7 +519,9 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath( + *store.getFSAccessor(), CanonPath { store.printStorePath(path) }, + FileIngestionMethod::Recursive, info->narHash.type); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a5f7cdf8124..39d60ae415f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -50,6 +50,18 @@ std::string ContentAddressMethod::render(HashType ht) const }, raw); } +FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const +{ + return std::visit(overloaded { + [&](const TextIngestionMethod & th) { + return FileIngestionMethod::Flat; + }, + [&](const FileIngestionMethod & fim) { + return fim; + } + }, raw); +} + std::string ContentAddress::render() const { return std::visit(overloaded { @@ -176,13 +188,13 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con }, ca.method.raw); } -std::optional ContentAddressWithReferences::fromPartsOpt( - ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept +ContentAddressWithReferences ContentAddressWithReferences::fromParts( + ContentAddressMethod method, Hash hash, StoreReferences refs) { return std::visit(overloaded { - [&](TextIngestionMethod _) -> std::optional { + [&](TextIngestionMethod _) -> ContentAddressWithReferences { if (refs.self) - return std::nullopt; + throw Error("self-reference not allowed with text hashing"); return ContentAddressWithReferences { TextInfo { .hash = std::move(hash), @@ -190,7 +202,7 @@ std::optional ContentAddressWithReferences::fromPa } }; }, - [&](FileIngestionMethod m2) -> std::optional { + [&](FileIngestionMethod m2) -> ContentAddressWithReferences { return ContentAddressWithReferences { FixedOutputInfo { .method = m2, diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index bdb558907b6..f04da03a738 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,6 +4,7 @@ #include #include "hash.hh" #include "path.hh" +#include "file-content-address.hh" #include "comparator.hh" #include "variant-wrapper.hh" @@ -31,22 +32,6 @@ namespace nix { */ struct TextIngestionMethod : std::monostate { }; -/** - * An enumeration of the main ways we can serialize file system - * objects. - */ -enum struct FileIngestionMethod : uint8_t { - /** - * Flat-file hashing. Directly ingest the contents of a single file - */ - Flat = 0, - /** - * Recursive (or NAR) hashing. Serializes the file-system object in Nix - * Archive format and ingest that - */ - Recursive = 1 -}; - /** * Compute the prefix to the hash algorithm which indicates how the * files were ingested. @@ -54,7 +39,7 @@ enum struct FileIngestionMethod : uint8_t { std::string makeFileIngestionPrefix(FileIngestionMethod m); /** - * An enumeration of all the ways we can serialize file system objects. + * An enumeration of all the ways we can content-address store objects. * * Just the type of a content address. Combine with the hash itself, and * we have a `ContentAddress` as defined below. Combine that, in turn, @@ -103,6 +88,14 @@ struct ContentAddressMethod * The rough inverse of `parse()`. */ std::string render(HashType ht) const; + + /** + * Get the underlying way to content-address file system objects. + * + * Different ways of hashing store objects may use the same method + * for hashing file systeme objects. + */ + FileIngestionMethod getFileIngestionMethod() const; }; @@ -266,11 +259,12 @@ struct ContentAddressWithReferences * * @param refs References to other store objects or oneself. * - * Do note that not all combinations are supported; `nullopt` is - * returns for invalid combinations. + * @note note that all combinations are supported. This is a + * *partial function* and exceptions will be thrown for invalid + * combinations. */ - static std::optional fromPartsOpt( - ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept; + static ContentAddressWithReferences fromParts( + ContentAddressMethod method, Hash hash, StoreReferences refs); ContentAddressMethod getMethod() const; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index be9b0b0d3f2..7f3881bc90d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -403,22 +403,9 @@ static void performOp(TunnelLogger * logger, ref store, auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr); auto hashType = hashType_; // work around clang bug FramedSource source(from); - // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. - return std::visit(overloaded { - [&](const TextIngestionMethod &) { - if (hashType != htSHA256) - throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", - name, printHashType(hashType)); - // We could stream this by changing Store - std::string contents = source.drain(); - auto path = store->addTextToStore(name, contents, refs, repair); - return store->queryPathInfo(path); - }, - [&](const FileIngestionMethod & fim) { - auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs); - return store->queryPathInfo(path); - }, - }, contentAddressMethod.raw); + // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store. + auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashType, refs, repair); + return store->queryPathInfo(path); }(); logger->stopWork(); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7314573547f..6c54ced5533 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -268,12 +268,13 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor StorePath addToStore( std::string_view name, - const Path & srcPath, - FileIngestionMethod method, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, HashType hashAlgo, + const StorePathSet & references, PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + RepairFlag repair) override { unsupported("addToStore"); } StorePath addTextToStore( diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4ff75f528c3..b6039081d2d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -12,6 +12,7 @@ #include "compression.hh" #include "signals.hh" #include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" #include #include @@ -1088,11 +1089,22 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (info.ca) { auto & specified = *info.ca; - auto actualHash = hashCAPath( - specified.method, - specified.hash.type, - info.path - ); + auto actualHash = ({ + HashModuloSink caSink { + specified.hash.type, + std::string { info.path.hashPart() }, + }; + PosixSourceAccessor accessor; + dumpPath( + *getFSAccessor(false), + CanonPath { printStorePath(info.path) }, + caSink, + specified.method.getFileIngestionMethod()); + ContentAddress { + .method = specified.method, + .hash = caSink.finish().first, + }; + }); if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), @@ -1115,8 +1127,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) +StorePath LocalStore::addToStoreFromDump( + Source & source0, + std::string_view name, + ContentAddressMethod method, + HashType hashAlgo, + const StorePathSet & references, + RepairFlag repair) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1166,25 +1183,21 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; - if (method == FileIngestionMethod::Recursive) - restorePath(tempPath, bothSource); - else - writeFile(tempPath, bothSource); + restorePath(tempPath, bothSource, method.getFileIngestionMethod()); dump.clear(); } auto [hash, size] = hashSink->finish(); - ContentAddressWithReferences desc = FixedOutputInfo { - .method = method, - .hash = hash, - .references = { + auto desc = ContentAddressWithReferences::fromParts( + method, + hash, + { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, - }, - }; + }); auto dstPath = makeFixedOutputPathFromCA(name, desc); @@ -1207,11 +1220,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name if (inMemory) { StringSource dumpSource { dump }; - /* Restore from the NAR in memory. */ - if (method == FileIngestionMethod::Recursive) - restorePath(realPath, dumpSource); - else - writeFile(realPath, dumpSource); + /* Restore from the buffer in memory. */ + restorePath(realPath, dumpSource, method.getFileIngestionMethod()); } else { /* Move the temporary path we restored above. */ moveFile(tempPath, realPath); @@ -1389,7 +1399,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false); + PosixSourceAccessor accessor; + std::string hash = hashPath( + accessor, CanonPath { linkPath }, + FileIngestionMethod::Recursive, htSHA256).first.to_string(HashFormat::Base32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1696,42 +1709,6 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } } -ContentAddress LocalStore::hashCAPath( - const ContentAddressMethod & method, const HashType & hashType, - const StorePath & path) -{ - return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); -} - -ContentAddress LocalStore::hashCAPath( - const ContentAddressMethod & method, - const HashType & hashType, - const Path & path, - const std::string_view pathHash -) -{ - HashModuloSink caSink ( hashType, std::string(pathHash) ); - std::visit(overloaded { - [&](const TextIngestionMethod &) { - readFile(path, caSink); - }, - [&](const FileIngestionMethod & m2) { - switch (m2) { - case FileIngestionMethod::Recursive: - dumpPath(path, caSink); - break; - case FileIngestionMethod::Flat: - readFile(path, caSink); - break; - } - }, - }, method.raw); - return ContentAddress { - .method = method, - .hash = caSink.finish().first, - }; -} - void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) { assert(drvPath.isDerivation()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8f0ffd2a2be..8bec2df2225 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -177,8 +177,13 @@ public: void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashType hashAlgo, + const StorePathSet & references, + RepairFlag repair) override; StorePath addTextToStore( std::string_view name, @@ -350,19 +355,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - // XXX: Make a generic `Store` method - ContentAddress hashCAPath( - const ContentAddressMethod & method, - const HashType & hashType, - const StorePath & path); - - ContentAddress hashCAPath( - const ContentAddressMethod & method, - const HashType & hashType, - const Path & path, - const std::string_view pathHash - ); - void addBuildLog(const StorePath & drvPath, std::string_view log) override; friend struct LocalDerivationGoal; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 0fa977545c7..7619974825a 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "signals.hh" #include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" #include #include @@ -146,7 +147,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path).first; + Hash hash = ({ + PosixSourceAccessor accessor; + hashPath( + accessor, CanonPath { path }, + FileIngestionMethod::Recursive, htSHA256).first; + }); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); /* Check if this is a known hash. */ @@ -156,7 +162,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, if (pathExists(linkPath)) { auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size - || (repair && hash != hashPath(htSHA256, linkPath).first)) + || (repair && hash != ({ + PosixSourceAccessor accessor; + hashPath( + accessor, CanonPath { linkPath }, + FileIngestionMethod::Recursive, htSHA256).first; + }))) { // XXX: Consider overwriting linkPath with our valid version. warn("removing corrupted link '%s'", linkPath); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f16949f42d6..79ab42e92e8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -502,8 +502,13 @@ ref RemoteStore::addCAToStore( } -StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) +StorePath RemoteStore::addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashType hashType, + const StorePathSet & references, + RepairFlag repair) { return addCAToStore(dump, name, method, hashType, references, repair)->path; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 1cc11af8662..e4a5704bf92 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -82,10 +82,15 @@ public: RepairFlag repair); /** - * Add a content-addressable store path. Does not support references. `dump` will be drained. + * Add a content-addressable store path. `dump` will be drained. */ - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = StorePathSet(), + RepairFlag repair = NoRepair) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0f88d9b92f0..2d87e671e52 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -230,22 +230,28 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA } -std::pair Store::computeStorePathFromDump( - Source & dump, +std::pair Store::computeStorePath( std::string_view name, - FileIngestionMethod method, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method, HashType hashAlgo, - const StorePathSet & references) const + const StorePathSet & references, + PathFilter & filter) const { - HashSink sink(hashAlgo); - dump.drainInto(sink); - auto h = sink.finish().first; - FixedOutputInfo caInfo { - .method = method, - .hash = h, - .references = {}, + auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first; + return { + makeFixedOutputPathFromCA( + name, + ContentAddressWithReferences::fromParts( + method, + h, + { + .others = references, + .self = false, + })), + h, }; - return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -263,21 +269,18 @@ StorePath Store::computeStorePathForText( StorePath Store::addToStore( std::string_view name, - const Path & _srcPath, - FileIngestionMethod method, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method, HashType hashAlgo, + const StorePathSet & references, PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + RepairFlag repair) { - Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - readFile(srcPath, sink); + dumpPath(accessor, path, sink, method.getFileIngestionMethod(), filter); }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); + return addToStoreFromDump(*source, name, method, hashAlgo, references, repair); } void Store::addMultipleToStore( @@ -402,8 +405,12 @@ digraph graphname { fileSink -> caHashSink } */ -ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, +ValidPathInfo Store::addToStoreSlow( + std::string_view name, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, HashType hashAlgo, + const StorePathSet & references, std::optional expectedCAHash) { HashSink narHashSink { htSHA256 }; @@ -423,7 +430,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, srcPath. The fact that we use scratchpadSink as a temporary buffer here is an implementation detail. */ auto fileSource = sinkToSource([&](Sink & scratchpadSink) { - dumpPath(srcPath, scratchpadSink); + accessor.dumpPath(srcPath, scratchpadSink); }); /* tapped provides the same data as fileSource, but we also write all the @@ -431,9 +438,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, TeeSource tapped { *fileSource, narSink }; NullParseSink blank; - auto & parseSink = method == FileIngestionMethod::Flat + auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat ? (ParseSink &) fileSink - : (ParseSink &) blank; + : method.getFileIngestionMethod() == FileIngestionMethod::Recursive + ? (ParseSink &) blank + : (abort(), (ParseSink &)*(ParseSink *)nullptr); // handled both cases /* The information that flows from tapped (besides being replicated in narSink), is now put in parseSink. */ @@ -450,21 +459,24 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, if (expectedCAHash && expectedCAHash != hash) throw Error("hash mismatch for '%s'", srcPath); + ValidPathInfo info { *this, name, - FixedOutputInfo { - .method = method, - .hash = hash, - .references = {}, - }, + ContentAddressWithReferences::fromParts( + method, + hash, + { + .others = references, + .self = false, + }), narHash, }; info.narSize = narSize; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { - dumpPath(srcPath, scratchpadSink); + accessor.dumpPath(srcPath, scratchpadSink); }); addToStore(info, *source); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 32ad2aa448b..a7f0838f447 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -291,15 +291,17 @@ public: StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** - * Read-only variant of addToStoreFromDump(). It returns the store - * path to which a NAR or flat file would be written. + * Read-only variant of addToStore(). It returns the store + * path for the given file sytem object. */ - std::pair computeStorePathFromDump( - Source & dump, + std::pair computeStorePath( std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - const StorePathSet & references = {}) const; + const StorePathSet & references = {}, + PathFilter & filter = defaultPathFilter) const; /** * Preparatory part of addTextToStore(). @@ -519,20 +521,26 @@ public: */ virtual StorePath addToStore( std::string_view name, - const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + const StorePathSet & references = StorePathSet(), PathFilter & filter = defaultPathFilter, - RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()); + RepairFlag repair = NoRepair); /** * Copy the contents of a path to the store and register the * validity the resulting path, using a constant amount of * memory. */ - ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + ValidPathInfo addToStoreSlow( + std::string_view name, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = StorePathSet(), std::optional expectedCAHash = {}); /** @@ -544,9 +552,13 @@ public: * * \todo remove? */ - virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()) + virtual StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = StorePathSet(), + RepairFlag repair = NoRepair) { unsupported("addToStoreFromDump"); } /** diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc new file mode 100644 index 00000000000..92d6ffae17d --- /dev/null +++ b/src/libutil/file-content-address.cc @@ -0,0 +1,49 @@ +#include "file-content-address.hh" +#include "archive.hh" + +namespace nix { + +void dumpPath( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + FileIngestionMethod method, + PathFilter & filter) +{ + switch (method) { + case FileIngestionMethod::Flat: + accessor.readFile(path, sink); + break; + case FileIngestionMethod::Recursive: + accessor.dumpPath(path, sink, filter); + break; + } +} + + +void restorePath( + const Path & path, + Source & source, + FileIngestionMethod method) +{ + switch (method) { + case FileIngestionMethod::Flat: + writeFile(path, source); + break; + case FileIngestionMethod::Recursive: + restorePath(path, source); + break; + } +} + + +HashResult hashPath( + SourceAccessor & accessor, const CanonPath & path, + FileIngestionMethod method, HashType ht, + PathFilter & filter) +{ + HashSink sink { ht }; + dumpPath(accessor, path, sink, method, filter); + return sink.finish(); +} + +} diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh new file mode 100644 index 00000000000..1ff8d5096b2 --- /dev/null +++ b/src/libutil/file-content-address.hh @@ -0,0 +1,56 @@ +#pragma once +///@file + +#include "source-accessor.hh" +#include "fs-sink.hh" +#include "util.hh" + +namespace nix { + +/** + * An enumeration of the main ways we can serialize file system + * objects. + */ +enum struct FileIngestionMethod : uint8_t { + /** + * Flat-file hashing. Directly ingest the contents of a single file + */ + Flat = 0, + /** + * Recursive (or NAR) hashing. Serializes the file-system object in + * Nix Archive format and ingest that. + */ + Recursive = 1, +}; + +/** + * Dump a serialization of the given file system object. + */ +void dumpPath( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + FileIngestionMethod method, + PathFilter & filter = defaultPathFilter); + +/** + * Restore a serialization of the given file system object. + * + * @TODO use an arbitrary `ParseSink`. + */ +void restorePath( + const Path & path, + Source & source, + FileIngestionMethod method); + +/** + * Compute the hash of the given file system object according to the + * given method. + * + * The hash is defined as (essentially) hashString(ht, dumpPath(path)). + */ +HashResult hashPath( + SourceAccessor & accessor, const CanonPath & path, + FileIngestionMethod method, HashType ht, + PathFilter & filter = defaultPathFilter); + +} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 144f7ae7ed4..a239fa62d80 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -366,15 +366,6 @@ HashResult HashSink::currentHash() } -HashResult hashPath( - HashType ht, const Path & path, PathFilter & filter) -{ - HashSink sink(ht); - dumpPath(path, sink, filter); - return sink.finish(); -} - - Hash compressHash(const Hash & hash, unsigned int newSize) { Hash h(hash.type); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 6ade6555c89..558c394a40a 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -167,14 +167,11 @@ Hash hashString(HashType ht, std::string_view s); Hash hashFile(HashType ht, const Path & path); /** - * Compute the hash of the given path, serializing as a Nix Archive and - * then hashing that. + * The final hash and the number of bytes digested. * - * The hash is defined as (essentially) hashString(ht, dumpPath(path)). + * @todo Convert to proper struct */ typedef std::pair HashResult; -HashResult hashPath(HashType ht, const Path & path, - PathFilter & filter = defaultPathFilter); /** * Compress a hash to the specified number of bytes by cyclically diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 25f0107bc4f..be0bd39fb9f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -13,6 +13,7 @@ #include "shared.hh" #include "graphml.hh" #include "legacy.hh" +#include "posix-source-accessor.hh" #include "path-with-outputs.hh" #include "posix-fs-canonicalise.hh" @@ -175,8 +176,12 @@ static void opAdd(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); + PosixSourceAccessor accessor; for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i))); + cout << fmt("%s\n", store->printStorePath(store->addToStore( + std::string(baseNameOf(i)), + accessor, + CanonPath::fromCwd(i)))); } @@ -196,8 +201,14 @@ static void opAddFixed(Strings opFlags, Strings opArgs) HashType hashAlgo = parseHashType(opArgs.front()); opArgs.pop_front(); + PosixSourceAccessor accessor; for (auto & i : opArgs) - std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path)); + std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow( + baseNameOf(i), + accessor, + CanonPath::fromCwd(i), + method, + hashAlgo).path)); } @@ -541,7 +552,10 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (canonicalise) canonicalisePathMetaData(store->printStorePath(info->path), {}); if (!hashGiven) { - HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); + HashResult hash = hashPath( + *store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }, + + FileIngestionMethod::Recursive, htSHA256); info->narHash = hash.first; info->narSize = hash.second; } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f9d487adab7..2249287083b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "store-api.hh" #include "archive.hh" +#include "posix-source-accessor.hh" using namespace nix; @@ -20,7 +21,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; + ContentAddressMethod caMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -48,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand )", .labels = {"hash-mode"}, .handler = {[this](std::string s) { - this->ingestionMethod = parseIngestionMethod(s); + this->caMethod = parseIngestionMethod(s); }}, }); } @@ -57,36 +58,17 @@ struct CmdAddToStore : MixDryRun, StoreCommand { if (!namePart) namePart = baseNameOf(path); - StringSink sink; - dumpPath(path, sink); - - auto narHash = hashString(htSHA256, sink.s); - - Hash hash = narHash; - if (ingestionMethod == FileIngestionMethod::Flat) { - HashSink hsink(htSHA256); - readFile(path, hsink); - hash = hsink.finish().first; - } - - ValidPathInfo info { - *store, - std::move(*namePart), - FixedOutputInfo { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - .references = {}, - }, - narHash, - }; - info.narSize = sink.s.size(); - - if (!dryRun) { - auto source = StringSource(sink.s); - store->addToStore(info, source); - } - - logger->cout("%s", store->printStorePath(info.path)); + PosixSourceAccessor accessor; + + auto path2 = CanonPath::fromCwd(path); + + auto storePath = dryRun + ? store->computeStorePath( + *namePart, accessor, path2, caMethod, htSHA256, {}).first + : store->addToStoreSlow( + *namePart, accessor, path2, caMethod, htSHA256, {}).path; + + logger->cout("%s", store->printStorePath(storePath)); } }; @@ -110,7 +92,7 @@ struct CmdAddFile : CmdAddToStore { CmdAddFile() { - ingestionMethod = FileIngestionMethod::Flat; + caMethod = FileIngestionMethod::Flat; } std::string description() override diff --git a/src/nix/hash.cc b/src/nix/hash.cc index d6595dcca05..7f15fe97387 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -5,6 +5,7 @@ #include "shared.hh" #include "references.hh" #include "archive.hh" +#include "posix-source-accessor.hh" using namespace nix; @@ -83,14 +84,8 @@ struct CmdHashBase : Command else hashSink = std::make_unique(ht); - switch (mode) { - case FileIngestionMethod::Flat: - readFile(path, *hashSink); - break; - case FileIngestionMethod::Recursive: - dumpPath(path, *hashSink); - break; - } + PosixSourceAccessor accessor; + dumpPath(accessor, CanonPath::fromCwd(path), *hashSink, mode); Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3ed7946a8b8..36522ed0c98 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -9,6 +9,7 @@ #include "attr-path.hh" #include "eval-inline.hh" #include "legacy.hh" +#include "posix-source-accessor.hh" #include @@ -122,7 +123,11 @@ std::tuple prefetchFile( Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url)); - auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + PosixSourceAccessor accessor; + auto info = store->addToStoreSlow( + *name, + accessor, CanonPath::fromCwd(tmpFile), + ingestionMethod, hashType, {}, expectedHash); storePath = info.path; assert(info.ca); hash = info.ca->hash;