Organize content addressing, use SourceAccessor with `Store::addToS…

…tore` Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
NixOS · Nov 20, 2023 · c6af435 · c6af435
1 parent cfc6494
commit c6af435
Show file tree

Hide file tree

Showing 31 changed files with 440 additions and 341 deletions.
diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc
@@ -44,7 +44,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = v.path().fetchToStore(state->store);
+        auto storePath = v.path().fetchToStore(*state->store);
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
@@ -2342,7 +2342,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     auto dstPath = i != srcToStore.end()
         ? i->second
         : [&]() {
-            auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
+            auto dstPath = path.fetchToStore(*store, path.baseName(), FileIngestionMethod::Recursive, defaultPathFilter, repair);
             allowPath(dstPath);
             srcToStore.insert_or_assign(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
@@ -2241,7 +2241,7 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair);
+            auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(*state.store, name, method, filter, state.repair);
             if (expectedHash && expectedStorePath != dstPath)
                 state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
             state.allowAndSetStorePathString(dstPath, v);

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
@@ -369,7 +369,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
 std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
 {
     auto [accessor, input2] = getAccessor(store, input);
-    auto storePath = accessor->root().fetchToStore(store, input2.getName());
+    auto storePath = accessor->root().fetchToStore(*store, input2.getName());
     return {storePath, input2};
 }
 

diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
@@ -4,28 +4,21 @@
 namespace nix {
 
 StorePath InputAccessor::fetchToStore(
-    ref<Store> store,
+    Store & store,
     const CanonPath & path,
     std::string_view name,
-    FileIngestionMethod method,
-    PathFilter * filter,
+    ContentAddressMethod method,
+    PathFilter & filter,
     RepairFlag repair)
 {
     Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
 
-    auto source = sinkToSource([&](Sink & sink) {
-        if (method == FileIngestionMethod::Recursive)
-            dumpPath(path, sink, filter ? *filter : defaultPathFilter);
-        else
-            readFile(path, sink);
-    });
-
-    auto storePath =
+    return
         settings.readOnlyMode
-        ? store->computeStorePathFromDump(*source, name, method, htSHA256).first
-        : store->addToStoreFromDump(*source, name, method, htSHA256, repair);
-
-    return storePath;
+        ? store.computeStorePath(
+            name, *this, path, method, htSHA256, {}, filter).first
+        : store.addToStore(
+            name, *this, path, method, htSHA256, {}, filter, repair);
 }
 
 SourcePath InputAccessor::root()
@@ -40,10 +33,10 @@ std::ostream & operator << (std::ostream & str, const SourcePath & path)
 }
 
 StorePath SourcePath::fetchToStore(
-    ref<Store> store,
+    Store & store,
     std::string_view name,
-    FileIngestionMethod method,
-    PathFilter * filter,
+    ContentAddressMethod method,
+    PathFilter & filter,
     RepairFlag repair) const
 {
     return accessor->fetchToStore(store, path, name, method, filter, repair);

diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh
@@ -28,11 +28,11 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<Inpu
     }
 
     StorePath fetchToStore(
-        ref<Store> store,
+        Store & store,
         const CanonPath & path,
         std::string_view name = "source",
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
-        PathFilter * filter = nullptr,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        PathFilter & filter = defaultPathFilter,
         RepairFlag repair = NoRepair);
 
     SourcePath root();
@@ -111,10 +111,10 @@ struct SourcePath
      * Copy this `SourcePath` to the Nix store.
      */
     StorePath fetchToStore(
-        ref<Store> store,
+        Store & store,
         std::string_view name = "source",
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
-        PathFilter * filter = nullptr,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        PathFilter & filter = defaultPathFilter,
         RepairFlag repair = NoRepair) const;
 
     /**

diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
@@ -6,6 +6,7 @@
 #include "tarfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"
+#include "posix-source-accessor.hh"
 
 #include "fetch-settings.hh"
 
@@ -210,7 +211,12 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
+                PosixSourceAccessor accessor;
+                auto storePath = store->addToStore(
+                    input.getName(),
+                    accessor, CanonPath { actualPath },
+                    FileIngestionMethod::Recursive, htSHA256, {},
+                    filter);
 
                 return {std::move(storePath), input};
             }
@@ -315,7 +321,8 @@ struct MercurialInputScheme : InputScheme
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
-        auto storePath = store->addToStore(name, tmpDir);
+        PosixSourceAccessor accessor;
+        auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir });
 
         Attrs infoAttrs({
             {"rev", input.getRev()->gitRev()},

diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
@@ -8,6 +8,7 @@
 #include "tarfile.hh"
 #include "types.hh"
 #include "split.hh"
+#include "posix-source-accessor.hh"
 
 namespace nix::fetchers {
 
@@ -156,7 +157,8 @@ DownloadTarballResult downloadTarball(
             throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
         auto topDir = tmpDir + "/" + members.begin()->name;
         lastModified = lstat(topDir).st_mtime;
-        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
+        PosixSourceAccessor accessor;
+        unpackedStorePath = store->addToStore(name, accessor, CanonPath { topDir }, FileIngestionMethod::Recursive, htSHA256, {}, defaultPathFilter, NoRepair);
     }
 
     Attrs infoAttrs({

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
@@ -300,24 +300,28 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     }});
 }
 
-StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
-    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
+StorePath BinaryCacheStore::addToStoreFromDump(
+    Source & dump,
+    std::string_view name,
+    ContentAddressMethod method,
+    HashType hashAlgo,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
     if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
         unsupported("addToStoreFromDump");
     return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
-            FixedOutputInfo {
-                .method = method,
-                .hash = nar.first,
-                .references = {
+            ContentAddressWithReferences::fromParts(
+                method,
+                nar.first,
+                {
                     .others = references,
                     // caller is not capable of creating a self-reference, because this is content-addressed without modulus
                     .self = false,
-                },
-            },
+                }),
             nar.first,
         };
         info.narSize = nar.second;
@@ -400,41 +404,35 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 
 StorePath BinaryCacheStore::addToStore(
     std::string_view name,
-    const Path & srcPath,
-    FileIngestionMethod method,
+    SourceAccessor & accessor,
+    const CanonPath & path,
+    ContentAddressMethod method,
     HashType hashAlgo,
+    const StorePathSet & references,
     PathFilter & filter,
-    RepairFlag repair,
-    const StorePathSet & references)
+    RepairFlag repair)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
        implementation of this method in terms of addToStoreFromDump. */
 
-    HashSink sink { hashAlgo };
-    if (method == FileIngestionMethod::Recursive) {
-        dumpPath(srcPath, sink, filter);
-    } else {
-        readFile(srcPath, sink);
-    }
-    auto h = sink.finish().first;
+    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
 
     auto source = sinkToSource([&](Sink & sink) {
-        dumpPath(srcPath, sink, filter);
+        accessor.dumpPath(path, sink, filter);
     });
     return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
-            FixedOutputInfo {
-                .method = method,
-                .hash = h,
-                .references = {
+            ContentAddressWithReferences::fromParts(
+                method,
+                h,
+                {
                     .others = references,
                     // caller is not capable of creating a self-reference, because this is content-addressed without modulus
                     .self = false,
-                },
-            },
+                }),
             nar.first,
         };
         info.narSize = nar.second;

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
@@ -123,17 +123,23 @@ public:
     void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
-    StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
+    StorePath addToStoreFromDump(
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method,
+        HashType hashAlgo,
+        const StorePathSet & references,
+        RepairFlag repair) override;
 
     StorePath addToStore(
         std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method,
+        SourceAccessor & accessor,
+        const CanonPath & srcPath,
+        ContentAddressMethod method,
         HashType hashAlgo,
+        const StorePathSet & references,
         PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references) override;
+        RepairFlag repair) override;
 
     StorePath addTextToStore(
         std::string_view name,