From dbca599c39edae664ec5891a6d7d8c90e0958382 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:00:45 -0500 Subject: [PATCH 001/102] Add ingestionMethodPrefix to get prefix This simplifies code so that references to r: come from ingestionMethodPrefix. --- src/libexpr/primops.cc | 3 +-- src/libstore/path.hh | 10 ++++++++++ src/libstore/store-api.cc | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d458ab2721f..4c72da44ad5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -725,8 +725,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { std::move(outPath), - (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") - + printHashType(h.type), + ingestionMethodPrefix(ingestionMethod) + printHashType(h.type), h.to_string(Base16, false), }); } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 5122e742290..4e670406eb1 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -78,6 +78,16 @@ enum struct FileIngestionMethod : uint8_t { Recursive = true }; +inline std::string ingestionMethodPrefix(FileIngestionMethod method) { + switch (method) { + case FileIngestionMethod::Flat: + return ""; + case FileIngestionMethod::Recursive: + return "r:"; + } + throw; +} + struct StorePathWithOutputs { StorePath path; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d479b86cb11..81cdaaaff23 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -172,20 +172,20 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - FileIngestionMethod recursive, + FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { + if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + ingestionMethodPrefix(method) + hash.to_string(Base16) + ":"), name); } @@ -787,15 +787,17 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const } else if (hasPrefix(ca, "fixed:")) { - FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6)); + FileIngestionMethod method = FileIngestionMethod::Flat; + if (ca.compare(6, 2, "r:") == 0) + method = FileIngestionMethod::Recursive; + Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length())); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; refs.erase(path); } - if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path) + if (store.makeFixedOutputPath(method, hash, path.name(), refs, hasSelfReference) == path) return true; else warn(); @@ -832,11 +834,9 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) { - return "fixed:" - + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") - + hash.to_string(); + return "fixed:" + ingestionMethodPrefix(method) + hash.to_string(); } From 9b452c965277abe43f53943dd52049bd89232bbf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:06:48 -0500 Subject: [PATCH 002/102] Handle g: prefix in nix store paths --- src/libstore/derivations.cc | 3 +++ src/libstore/path.hh | 7 +++++-- src/libstore/store-api.cc | 6 ++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c68e7b16b1a..793eae4a223 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -17,6 +17,9 @@ void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & has if (string(algo, 0, 2) == "r:") { recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); + } else if (string(algo, 0, 2) == "g:") { + recursive = FileIngestionMethod::Git; + algo = string(algo, 2); } HashType hashType = parseHashType(algo); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 4e670406eb1..052d9a6b3de 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -74,8 +74,9 @@ const size_t storePathHashLen = 32; // i.e. 160 bits const std::string drvExtension = ".drv"; enum struct FileIngestionMethod : uint8_t { - Flat = false, - Recursive = true + Flat, + Recursive, + Git }; inline std::string ingestionMethodPrefix(FileIngestionMethod method) { @@ -84,6 +85,8 @@ inline std::string ingestionMethodPrefix(FileIngestionMethod method) { return ""; case FileIngestionMethod::Recursive: return "r:"; + case FileIngestionMethod::Git: + return "g:"; } throw; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 81cdaaaff23..b128ed85b31 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -118,8 +118,8 @@ string storePathToHash(const Path & path) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:::", where - = "r:" for recursive (path) hashes, or "" for flat - (file) hashes + = "r:" for recursive (path) hashes, "g:" for git + paths, or "" for flat (file) hashes = "md5", "sha1" or "sha256" = base-16 representation of the path or flat hash of the contents of the path (or expected contents of the @@ -790,6 +790,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const FileIngestionMethod method = FileIngestionMethod::Flat; if (ca.compare(6, 2, "r:") == 0) method = FileIngestionMethod::Recursive; + else if (ca.compare(6, 2, "g:") == 0) + method = FileIngestionMethod::Git; Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length())); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; From 16c4945101bd51a27509edef7350b5417d4e7a1b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:45:42 -0500 Subject: [PATCH 003/102] Use git: instead of g: for ca prefix this is more precise --- src/libstore/derivations.cc | 2 +- src/libstore/path.hh | 2 +- src/libstore/store-api.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 793eae4a223..bef42312bdf 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -17,7 +17,7 @@ void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & has if (string(algo, 0, 2) == "r:") { recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); - } else if (string(algo, 0, 2) == "g:") { + } else if (string(algo, 0, 4) == "git:") { recursive = FileIngestionMethod::Git; algo = string(algo, 2); } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 052d9a6b3de..3374a47486c 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -86,7 +86,7 @@ inline std::string ingestionMethodPrefix(FileIngestionMethod method) { case FileIngestionMethod::Recursive: return "r:"; case FileIngestionMethod::Git: - return "g:"; + return "git:"; } throw; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b128ed85b31..ed8dbcbeabd 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -118,7 +118,7 @@ string storePathToHash(const Path & path) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:::", where - = "r:" for recursive (path) hashes, "g:" for git + = "r:" for recursive (path) hashes, "git:" for git paths, or "" for flat (file) hashes = "md5", "sha1" or "sha256" = base-16 representation of the path or flat hash of @@ -790,7 +790,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const FileIngestionMethod method = FileIngestionMethod::Flat; if (ca.compare(6, 2, "r:") == 0) method = FileIngestionMethod::Recursive; - else if (ca.compare(6, 2, "g:") == 0) + else if (ca.compare(6, 4, "git:") == 0) method = FileIngestionMethod::Git; Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length())); auto refs = cloneStorePathSet(references); From 6ba83609472faa95b8c27650378504c16dfa319c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 14:34:33 -0500 Subject: [PATCH 004/102] Rename recursive to method in more places --- src/libstore/derivations.cc | 8 ++++---- src/libstore/derivations.hh | 2 +- src/nix-store/nix-store.cc | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index bef42312bdf..acab418b63a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,16 +9,16 @@ namespace nix { -void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const +void DerivationOutput::parseHashInfo(FileIngestionMethod & method, Hash & hash) const { - recursive = FileIngestionMethod::Flat; + method = FileIngestionMethod::Flat; string algo = hashAlgo; if (string(algo, 0, 2) == "r:") { - recursive = FileIngestionMethod::Recursive; + method = FileIngestionMethod::Recursive; algo = string(algo, 2); } else if (string(algo, 0, 4) == "git:") { - recursive = FileIngestionMethod::Git; + method = FileIngestionMethod::Git; algo = string(algo, 2); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b1224b93bbb..705049d6fa4 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -22,7 +22,7 @@ struct DerivationOutput , hashAlgo(std::move(hashAlgo)) , hash(std::move(hash)) { } - void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; + void parseHashInfo(FileIngestionMethod & method, Hash & hash) const; }; typedef std::map DerivationOutputs; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3a3060ad82d..68fe6f72735 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.empty()) @@ -187,17 +187,17 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo))); + cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, method, hashAlgo))); } /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.size() != 3) @@ -208,7 +208,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(method, Hash(hash, hashAlgo), name))); } From 2e16c283b19c744a47710060fa34f4e8aedc8d7a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 14:36:06 -0500 Subject: [PATCH 005/102] Throw error when using unsupported git file ingestion --- src/libstore/binary-cache-store.cc | 2 ++ src/libstore/remote-store.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f13736c58c0..38c515f229c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -339,6 +339,8 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); + } else if (method == FileIngestionMethod::Git) { + throw Error("cannot add to binary cache store using the git file ingestion method"); } else { auto s = readFile(srcPath); dumpString(s, sink); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e699..4425ab9f07c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -488,6 +488,8 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + if (method == FileIngestionMethod::Git) throw Error("cannot remotely add to store using the git file ingestion method"); + auto conn(getConnection()); Path srcPath(absPath(_srcPath)); From 256bbb651f7b8d63354ba50597aeb0f7c9929e36 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 27 May 2020 19:30:53 -0400 Subject: [PATCH 006/102] First draft of git parsing --- src/libstore/local-store.cc | 14 ++++++-- src/libutil/archive.cc | 56 +------------------------------- src/libutil/git.cc | 52 +++++++++++++++++++++++++++++ src/libutil/git.hh | 15 +++++++++ src/libutil/restore-sink.hh | 65 +++++++++++++++++++++++++++++++++++++ src/libutil/serialise.cc | 4 +++ src/libutil/serialise.hh | 1 + 7 files changed, 149 insertions(+), 58 deletions(-) create mode 100644 src/libutil/git.cc create mode 100644 src/libutil/git.hh create mode 100644 src/libutil/restore-sink.hh diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b59106..f7af6ebb091 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1067,11 +1067,19 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (method == FileIngestionMethod::Recursive) { + switch method{ + case FileIngestionMethod::Flat: + writeFile(realPath, dump); + break; + case FileIngestionMethod::Recursive: StringSource source(dump); restorePath(realPath, source); - } else - writeFile(realPath, dump); + break; + case FileIngestionMethod::Git: + StringSource source(dump); + restoreGit(realPath, source); + break; + } canonicalisePathMetaData(realPath, -1); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index db544a212ab..a0aeb5f3965 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -12,6 +12,7 @@ #include #include "archive.hh" +#include "restore-sink.hh" #include "util.hh" #include "config.hh" @@ -294,61 +295,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError(format("creating directory '%1%'") % p); - }; - - void createRegularFile(const Path & path) - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError(format("creating file '%1%'") % p); - } - - void isExecutable() - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(unsigned long long len) - { -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError(format("preallocating file of %1% bytes") % len); - } -#endif - } - - void receiveContents(unsigned char * data, unsigned int len) - { - writeFull(fd.get(), data, len); - } - - void createSymlink(const Path & path, const string & target) - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - void restorePath(const Path & path, Source & source) { diff --git a/src/libutil/git.cc b/src/libutil/git.cc new file mode 100644 index 00000000000..7425eda4fa8 --- /dev/null +++ b/src/libutil/git.cc @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +#include // for strcasecmp + +#include +#include +#include +#include +#include + +#include "util.hh" +#include "config.hh" +#include "restore-sink.hh" + +#include "git.hh" + +namespace nix { + + +// Converts a Path to a ParseSink +void restoreGit(const Path & path, Source & source) { + + RestoreSink sink; + sink.dstPath = path; + parseDump(sink, source); + +} + +void parseGit(ParseSink & sink, Source & source) { + parse(sink, source, ""); +} + +static void parse(ParseSink & sink, Source & source, const Path & path) { + uint8_t buf[4]; + + std::basic_string_view buf_v { + (const uint8_t *) & buf, + std::size(buf) + }; + source(buf_v); + if (buf_v.compare((const uint8_t *)"blob")) { + uint8_t space; + source(& space, 1); + } + else { + } +} + +} diff --git a/src/libutil/git.hh b/src/libutil/git.hh new file mode 100644 index 00000000000..554bf41338f --- /dev/null +++ b/src/libutil/git.hh @@ -0,0 +1,15 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" +#include "archive.hh" + +namespace nix { + +void restoreGit(const Path & path, Source & source); + +void parseGit(ParseSink & sink, Source & source); + +static void parse(ParseSink & sink, Source & source, const Path & path); + +} diff --git a/src/libutil/restore-sink.hh b/src/libutil/restore-sink.hh new file mode 100644 index 00000000000..5fcb8c595ab --- /dev/null +++ b/src/libutil/restore-sink.hh @@ -0,0 +1,65 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" +#include "archive.hh" + +namespace nix { + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError(format("creating directory '%1%'") % p); + }; + + void createRegularFile(const Path & path) + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError(format("creating file '%1%'") % p); + } + + void isExecutable() + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(unsigned long long len) + { +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError(format("preallocating file of %1% bytes") % len); + } +#endif + } + + void receiveContents(unsigned char * data, unsigned int len) + { + writeFull(fd.get(), data, len); + } + + void createSymlink(const Path & path, const string & target) + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + + +} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8201549fd7d..79b6048e6ee 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -89,6 +89,10 @@ void Source::operator () (unsigned char * data, size_t len) } } +void Source::operator () (std::basic_string_view & data) +{ + (*this)((unsigned char *) data.data(), data.size()); +} std::string Source::drain() { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a04118512fc..7bcd476aafc 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -54,6 +54,7 @@ struct Source It blocks until all the requested data is available, or throws an error if it is not going to be available. */ void operator () (unsigned char * data, size_t len); + void operator () (std::basic_string_view & data); /* Store up to ‘len’ in the buffer pointed to by ‘data’, and return the number of bytes stored. It blocks until at least From 85eeb2acc7083de7f725d54798506125f100a8f5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 21:04:35 -0500 Subject: [PATCH 007/102] Use switch ingestion method in BinaryCacheStore::addToStore --- src/libstore/binary-cache-store.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 38c515f229c..27e7f1eb79b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -336,15 +336,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath small files. */ StringSink sink; Hash h; - if (method == FileIngestionMethod::Recursive) { + switch (method) { + case FileIngestionMethod::Recursive: dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); - } else if (method == FileIngestionMethod::Git) { - throw Error("cannot add to binary cache store using the git file ingestion method"); - } else { + break; + } + case FileIngestionMethod::Flat: auto s = readFile(srcPath); dumpString(s, sink); h = hashString(hashAlgo, s); + break; + case FileIngestionMethod::Git: + throw Error("cannot add to binary cache store using the git file ingestion method"); + default: throw; } ValidPathInfo info(makeFixedOutputPath(method, h, name)); From 39df335302d71e9e6de9dca4b5393b26b240b2b0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 18:55:30 -0500 Subject: [PATCH 008/102] More work on git objects --- src/libstore/binary-cache-store.cc | 5 +- src/libstore/local-store.cc | 9 ++- src/libstore/path.hh | 2 +- src/libutil/archive.cc | 1 - src/libutil/archive.hh | 15 +--- src/libutil/{restore-sink.hh => fs-sink.hh} | 14 +++- src/libutil/git.cc | 76 ++++++++++++++++++++- src/libutil/git.hh | 24 ++++++- src/nix/hash.cc | 40 +++++++---- tests/hash.sh | 15 ++++ 10 files changed, 162 insertions(+), 39 deletions(-) rename src/libutil/{restore-sink.hh => fs-sink.hh} (79%) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 27e7f1eb79b..5ca5bd3bb28 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -341,15 +341,14 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); break; - } - case FileIngestionMethod::Flat: + case FileIngestionMethod::Flat: { auto s = readFile(srcPath); dumpString(s, sink); h = hashString(hashAlgo, s); break; + } case FileIngestionMethod::Git: throw Error("cannot add to binary cache store using the git file ingestion method"); - default: throw; } ValidPathInfo info(makeFixedOutputPath(method, h, name)); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f7af6ebb091..39efa650b25 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,5 +1,6 @@ #include "local-store.hh" #include "globals.hh" +#include "git.hh" #include "archive.hh" #include "pathlocks.hh" #include "worker-protocol.hh" @@ -1067,19 +1068,21 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - switch method{ + switch (method) { case FileIngestionMethod::Flat: writeFile(realPath, dump); break; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::Recursive: { StringSource source(dump); restorePath(realPath, source); break; - case FileIngestionMethod::Git: + } + case FileIngestionMethod::Git: { StringSource source(dump); restoreGit(realPath, source); break; } + } canonicalisePathMetaData(realPath, -1); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 3374a47486c..68fcf50103e 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -76,7 +76,7 @@ const std::string drvExtension = ".drv"; enum struct FileIngestionMethod : uint8_t { Flat, Recursive, - Git + Git, }; inline std::string ingestionMethodPrefix(FileIngestionMethod method) { diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index a0aeb5f3965..3670131c291 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -12,7 +12,6 @@ #include #include "archive.hh" -#include "restore-sink.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 768fe25368d..14bf9740d91 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -2,7 +2,7 @@ #include "types.hh" #include "serialise.hh" - +#include "fs-sink.hh" namespace nix { @@ -50,19 +50,6 @@ void dumpPath(const Path & path, Sink & sink, void dumpString(const std::string & s, Sink & sink); -/* FIXME: fix this API, it sucks. */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void isExecutable() { }; - virtual void preallocateContents(unsigned long long size) { }; - virtual void receiveContents(unsigned char * data, unsigned int len) { }; - - virtual void createSymlink(const Path & path, const string & target) { }; -}; - struct TeeSink : ParseSink { TeeSource source; diff --git a/src/libutil/restore-sink.hh b/src/libutil/fs-sink.hh similarity index 79% rename from src/libutil/restore-sink.hh rename to src/libutil/fs-sink.hh index 5fcb8c595ab..c5599d27471 100644 --- a/src/libutil/restore-sink.hh +++ b/src/libutil/fs-sink.hh @@ -2,10 +2,22 @@ #include "types.hh" #include "serialise.hh" -#include "archive.hh" namespace nix { +/* FIXME: fix this API, it sucks. */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void isExecutable() { }; + virtual void preallocateContents(unsigned long long size) { }; + virtual void receiveContents(unsigned char * data, unsigned int len) { }; + + virtual void createSymlink(const Path & path, const string & target) { }; +}; + struct RestoreSink : ParseSink { Path dstPath; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 7425eda4fa8..32ba36677e4 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -13,19 +13,22 @@ #include "util.hh" #include "config.hh" -#include "restore-sink.hh" +#include "hash.hh" #include "git.hh" +using namespace std::string_literals; + namespace nix { +static void parse(ParseSink & sink, Source & source, const Path & path); // Converts a Path to a ParseSink void restoreGit(const Path & path, Source & source) { RestoreSink sink; sink.dstPath = path; - parseDump(sink, source); + parseGit(sink, source); } @@ -49,4 +52,73 @@ static void parse(ParseSink & sink, Source & source, const Path & path) { } } +// TODO stream file into sink, rather than reading into vector +GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink) +{ + auto s = (format("blob %d\0%s"s) % std::to_string(st.st_size) % readFile(path)).str(); + + vector v; + std::copy(s.begin(), s.end(), std::back_inserter(v)); + sink(v.data(), v.size()); + return st.st_mode & S_IXUSR + ? GitMode::Executable + : GitMode::Regular; +} + +GitMode dumpGitTree(const GitTree & entries, Sink & sink) +{ + std::string s1 = ""; + for (auto & i : entries) { + unsigned int mode; + switch (i.second.first) { + case GitMode::Directory: mode = 40000; break; + case GitMode::Executable: mode = 100755; break; + case GitMode::Regular: mode = 100644; break; + } + s1 += (format("%6d %s\0%s"s) % mode % i.first % i.second.second).str(); + } + + std::string s2 = (format("tree %d\0%s"s) % s1.size() % s1).str(); + + vector v; + std::copy(s2.begin(), s2.end(), std::back_inserter(v)); + sink(v.data(), v.size()); + return GitMode::Directory; +} + +// Returns the perm in addition +std::pair dumpGitHashInternal( + std::function()> makeHashSink, + const Path & path, PathFilter & filter) +{ + auto hashSink = makeHashSink(); + struct stat st; + GitMode perm; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + + if (S_ISREG(st.st_mode)) { + perm = dumpGitBlob(path, st, *hashSink); + } else if (S_ISDIR(st.st_mode)) { + GitTree entries; + for (auto & i : readDirectory(path)) + if (filter(path + "/" + i.name)) { + entries[i.name] = dumpGitHashInternal(makeHashSink, path + "/" + i.name, filter); + } + perm = dumpGitTree(entries, *hashSink); + } else { + throw Error(format("file '%1%' has an unsupported type") % path); + } + + auto hash = hashSink->finish().first; + return std::pair { perm, hash }; +} + +Hash dumpGitHash( + std::function()> makeHashSink, + const Path & path, PathFilter & filter) +{ + return dumpGitHashInternal(makeHashSink, path, filter).second; +} + } diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 554bf41338f..7b11f72be29 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -2,14 +2,34 @@ #include "types.hh" #include "serialise.hh" -#include "archive.hh" +#include "fs-sink.hh" namespace nix { +enum struct GitMode { + Directory, + Executable, + Regular, +}; + void restoreGit(const Path & path, Source & source); void parseGit(ParseSink & sink, Source & source); -static void parse(ParseSink & sink, Source & source, const Path & path); +// Dumps a single file to a sink +GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink); + +typedef std::map> GitTree; + +// Dumps a representation of a git tree to a sink +GitMode dumpGitTree(const GitTree & entries, Sink & sink); + +// Recursively dumps path, hashing as we go +Hash dumpGitHash( + std::function()>, + const Path & path, + PathFilter & filter = defaultPathFilter); +// N.B. There is no way to recursively dump to a sink, as that doesn't make +// sense with the git hash/data model where the information is Merklized. } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 0a7c343d0dd..4f3b3cd426f 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -4,6 +4,7 @@ #include "shared.hh" #include "references.hh" #include "archive.hh" +#include "git.hh" using namespace nix; @@ -41,6 +42,8 @@ struct CmdHash : Command d = "print cryptographic hash of a regular file"; case FileIngestionMethod::Recursive: d = "print cryptographic hash of the NAR serialisation of a path"; + case FileIngestionMethod::Git: + d = "print cryptographic hash of the Git serialisation of a path"; }; return d; } @@ -51,22 +54,34 @@ struct CmdHash : Command { for (auto path : paths) { - std::unique_ptr hashSink; - if (modulus) - hashSink = std::make_unique(ht, *modulus); - else - hashSink = std::make_unique(ht); + auto makeHashSink = [&]() -> std::unique_ptr { + std::unique_ptr t; + if (modulus) + t = std::make_unique(ht, *modulus); + else + t = std::make_unique(ht); + return t; + }; + Hash h; switch (mode) { - case FileIngestionMethod::Flat: + case FileIngestionMethod::Flat: { + auto hashSink = makeHashSink(); readFile(path, *hashSink); + h = hashSink->finish().first; break; - case FileIngestionMethod::Recursive: + } + case FileIngestionMethod::Recursive: { + auto hashSink = makeHashSink(); dumpPath(path, *hashSink); + h = hashSink->finish().first; + break; + } + case FileIngestionMethod::Git: + h = dumpGitHash(makeHashSink, path); break; } - Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); logger->stdout(h.to_string(base, base == SRI)); } @@ -75,6 +90,7 @@ struct CmdHash : Command static RegisterCommand r1("hash-file", [](){ return make_ref(FileIngestionMethod::Flat); }); static RegisterCommand r2("hash-path", [](){ return make_ref(FileIngestionMethod::Recursive); }); +static RegisterCommand r3("hash-git", [](){ return make_ref(FileIngestionMethod::Git); }); struct CmdToBase : Command { @@ -106,10 +122,10 @@ struct CmdToBase : Command } }; -static RegisterCommand r3("to-base16", [](){ return make_ref(Base16); }); -static RegisterCommand r4("to-base32", [](){ return make_ref(Base32); }); -static RegisterCommand r5("to-base64", [](){ return make_ref(Base64); }); -static RegisterCommand r6("to-sri", [](){ return make_ref(SRI); }); +static RegisterCommand r4("to-base16", [](){ return make_ref(Base16); }); +static RegisterCommand r5("to-base32", [](){ return make_ref(Base32); }); +static RegisterCommand r6("to-base64", [](){ return make_ref(Base64); }); +static RegisterCommand r7("to-sri", [](){ return make_ref(SRI); }); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) diff --git a/tests/hash.sh b/tests/hash.sh index 4cfc9790101..33b5bef7595 100644 --- a/tests/hash.sh +++ b/tests/hash.sh @@ -85,3 +85,18 @@ try3() { try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" + +# Git. +try4 () { + hash=$(nix hash-git --base16 --type sha1 $TEST_ROOT/hash-path) + if test "$hash" != "$1"; then + echo "git hash, expected $1, got $hash" + exit 1 + fi +} + +rm -rf $TEST_ROOT/hash-path +mkdir $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path/hello + +try4 "117c62a8c5e01758bd284126a6af69deab9dbbe2" From ffc1024139d6ba6aa50cff7cb382e64fec7cad4b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 11:42:07 -0500 Subject: [PATCH 009/102] Add fcntl.h to fs-sink.hh --- src/libutil/fs-sink.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index c5599d27471..d3d1e43b79f 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -1,5 +1,7 @@ #pragma once +#include + #include "types.hh" #include "serialise.hh" From 124cc892a16b251d1f4aa1094becb4fb441e0929 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 May 2020 18:38:17 -0400 Subject: [PATCH 010/102] Fix bugs with git hashing --- foo/hello.txt | 1 + src/libutil/git.cc | 6 +++--- src/nix/hash.cc | 5 ++++- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 foo/hello.txt diff --git a/foo/hello.txt b/foo/hello.txt new file mode 100644 index 00000000000..ef8f9fc5d88 --- /dev/null +++ b/foo/hello.txt @@ -0,0 +1 @@ +asdfsf diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 32ba36677e4..7e8f25b3e14 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -57,7 +57,7 @@ GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink) { auto s = (format("blob %d\0%s"s) % std::to_string(st.st_size) % readFile(path)).str(); - vector v; + vector v; std::copy(s.begin(), s.end(), std::back_inserter(v)); sink(v.data(), v.size()); return st.st_mode & S_IXUSR @@ -75,12 +75,12 @@ GitMode dumpGitTree(const GitTree & entries, Sink & sink) case GitMode::Executable: mode = 100755; break; case GitMode::Regular: mode = 100644; break; } - s1 += (format("%6d %s\0%s"s) % mode % i.first % i.second.second).str(); + s1 += (format("%6d %s\0%s"s) % mode % i.first % i.second.second.hash).str(); } std::string s2 = (format("tree %d\0%s"s) % s1.size() % s1).str(); - vector v; + vector v; std::copy(s2.begin(), s2.end(), std::back_inserter(v)); sink(v.data(), v.size()); return GitMode::Directory; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 4f3b3cd426f..3a8c82da17b 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -36,14 +36,17 @@ struct CmdHash : Command std::string description() override { - const char* d; + const char *d; switch (mode) { case FileIngestionMethod::Flat: d = "print cryptographic hash of a regular file"; + break; case FileIngestionMethod::Recursive: d = "print cryptographic hash of the NAR serialisation of a path"; + break; case FileIngestionMethod::Git: d = "print cryptographic hash of the Git serialisation of a path"; + break; }; return d; } From a1f57606958ab760ff06dd6571815f59e32918a5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 May 2020 18:54:51 -0400 Subject: [PATCH 011/102] Remove cached data --- foo/hello.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 foo/hello.txt diff --git a/foo/hello.txt b/foo/hello.txt deleted file mode 100644 index ef8f9fc5d88..00000000000 --- a/foo/hello.txt +++ /dev/null @@ -1 +0,0 @@ -asdfsf From 4ced63cd3b5f5be34465a0a46b44a0c3928e1c8a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 1 Jun 2020 16:03:48 -0400 Subject: [PATCH 012/102] Temporarily reduce CI platform's for sake of Obsidian's own CI Hopefully we can instead pass arguments like Hydra does, soon. --- release.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 2a320e1c3bc..23147d67654 100644 --- a/release.nix +++ b/release.nix @@ -1,7 +1,8 @@ { nix ? builtins.fetchGit ./. , nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz , officialRelease ? false -, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] +, systems ? [ "x86_64-linux" "x86_64-darwin" ] + # ^ Temporary shorten list for sake of Obsidian's CI. }: let From 130267de37e799f77f9f7d1edc8063126af9e640 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 17:22:40 -0500 Subject: [PATCH 013/102] Handle git ingestion over daemon --- src/libstore/daemon.cc | 19 ++++++++++++++++--- src/libstore/local-store.cc | 18 +++++++++++++++--- src/libstore/remote-store.cc | 2 +- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f1afdff699e..aa2cbe6d487 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -8,6 +8,7 @@ #include "archive.hh" #include "derivations.hh" #include "args.hh" +#include "git.hh" namespace nix::daemon { @@ -358,7 +359,8 @@ static void performOp(TunnelLogger * logger, ref store, std::string s, baseName; FileIngestionMethod method; { - bool fixed, recursive; + bool fixed; + unsigned char recursive; from >> baseName >> fixed /* obsolete */ >> recursive >> s; method = FileIngestionMethod { recursive }; /* Compatibility hack. */ @@ -372,14 +374,25 @@ static void performOp(TunnelLogger * logger, ref store, TeeSource savedNAR(from); RetrieveRegularNARSink savedRegular; - if (method == FileIngestionMethod::Recursive) { + switch (method) { + case FileIngestionMethod::Recursive: { /* Get the entire NAR dump from the client and save it to a string so that we can pass it to addToStoreFromDump(). */ ParseSink sink; /* null sink; just parse the NAR */ parseDump(sink, savedNAR); - } else + break; + } + case FileIngestionMethod::Git: { + ParseSink sink; + parseGit(sink, savedNAR); + break; + } + case FileIngestionMethod::Flat: { parseDump(savedRegular, from); + break; + } + } logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 39efa650b25..e214c334b73 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1017,7 +1017,10 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, return n; }); - restorePath(realPath, wrapperSource); + if (hasPrefix(info.ca, "fixed:git:")) + restoreGit(realPath, wrapperSource); + else + restorePath(realPath, wrapperSource); auto hashResult = hashSink->finish(); @@ -1122,10 +1125,19 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (method == FileIngestionMethod::Recursive) + switch (method) { + case FileIngestionMethod::Recursive: dumpPath(srcPath, sink, filter); - else + break; + case FileIngestionMethod::Git:{ + dumpGit(srcPath, sink, filter); + break; + } + case FileIngestionMethod::Flat: { sink.s = make_ref(readFile(srcPath)); + break; + } + } return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4425ab9f07c..842c2e17666 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -498,7 +498,7 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, << wopAddToStore << name << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (method == FileIngestionMethod::Recursive ? 1 : 0) + << (uint8_t) method << printHashType(hashAlgo); try { From df747447bf3cfd965fec37f53cac2c04fc8b5d4b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 17:23:05 -0500 Subject: [PATCH 014/102] Parse git blobs correctly This matches what we want for blobs. Trees are still in progress - we need a way to symlink to other objects, using that to determine ca. --- src/libutil/git.cc | 80 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 13 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 7e8f25b3e14..cf9b222ec80 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -36,20 +36,74 @@ void parseGit(ParseSink & sink, Source & source) { parse(sink, source, ""); } -static void parse(ParseSink & sink, Source & source, const Path & path) { - uint8_t buf[4]; - - std::basic_string_view buf_v { - (const uint8_t *) & buf, - std::size(buf) - }; - source(buf_v); - if (buf_v.compare((const uint8_t *)"blob")) { - uint8_t space; - source(& space, 1); - } - else { +string getStringUntil(Source & source, char byte) { + string s; + unsigned char n[1]; + source(n, 1); + while (*n != byte) { + s += *n; + source(n, 1); } + return s; +} + +string getString(Source & source, int n){ + std::vector v(n); + source(v.data(), n); + return std::string(v.begin(), v.end()); +} + +static void parse(ParseSink & sink, Source & source, const Path & path) { + auto type = getString(source, 5); + + if (type == "blob ") { + sink.createRegularFile(path); + + unsigned long long size = std::stoi(getStringUntil(source, 0)); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::vector buf(65536); + + while (left) { + checkInterrupt(); + auto n = buf.size(); + if ((unsigned long long)n > left) n = left; + source(buf.data(), n); + sink.receiveContents(buf.data(), n); + left -= n; + } + } else if (type == "tree ") { + unsigned long long size = std::stoi(getStringUntil(source, 0)); + unsigned long long left = size; + + sink.createDirectory(path); + + while (left) { + string perms = getStringUntil(source, ' '); + left -= perms.size(); + left -= 1; + + int perm = std::stoi(perms); + if (perm != 100644 && perm != 100755 && perm != 644 && perm != 755 && perm != 40000) + throw Error(format("Unknown Git permission: %d") % perm); + + // TODO: handle permissions somehow + + string name = getStringUntil(source, 0); + left -= name.size(); + left -= 1; + + string hashs = getString(source, 20); + left -= 20; + + Hash hash(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); + + sink.createSymlink(path + "/" + name, "../" + name); + } + } else throw Error("input doesn't look like a Git object"); } // TODO stream file into sink, rather than reading into vector From f292e50df72356a84d103e38e1447882d01dfca4 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 17:24:01 -0500 Subject: [PATCH 015/102] Add --git argument to add-to-store --- src/nix/add-to-store.cc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f43f774c1c8..c5b7c0368f7 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "store-api.hh" #include "archive.hh" +#include "git.hh" using namespace nix; @@ -9,6 +10,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; + bool git = false; CmdAddToStore() { @@ -21,6 +23,12 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "git", + .description = "treat path as a git object", + .handler = {&this->git, true}, + }); } std::string description() override @@ -40,15 +48,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand { if (!namePart) namePart = baseNameOf(path); + auto ingestionMethod = git ? FileIngestionMethod::Git : FileIngestionMethod::Recursive; + StringSink sink; - dumpPath(path, sink); + if (git) + dumpGit(path, sink); + else + dumpPath(path, sink); auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); + ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart)); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = makeFixedOutputCA(ingestionMethod, info.narHash); if (!dryRun) { auto source = StringSource { *sink.s }; From 1592d09a1fae82c6d7bb723f26b2429d427db9c3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 17:24:14 -0500 Subject: [PATCH 016/102] Add test for nix add-to-store --- tests/add.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/add.sh b/tests/add.sh index e26e05843d7..5ae4b1886e2 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -26,3 +26,7 @@ hash2=$(nix-hash --type sha256 --base32 ./dummy) echo $hash2 test "$hash1" = "sha256:$hash2" + +path5=$(nix add-to-store --git ./dummy) +hash3=$(nix-store -q --hash $path5) +test "$hash3" = "sha256:$(nix hash-git --base32 ./dummy)" From cf819912b40c4dea8c16c80b319108745de6b9f7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 18:24:45 -0500 Subject: [PATCH 017/102] Depend on install for installcheck --- mk/tests.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/tests.mk b/mk/tests.mk index 70c30661b95..c3c1f4c28dd 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -8,7 +8,7 @@ define run-install-test endef # Color code from https://unix.stackexchange.com/a/10065 -installcheck: +installcheck: install @total=0; failed=0; \ red=""; \ green=""; \ From 9e9c2ce71f3cc4688a45eb948769683be84ceece Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 11:48:09 -0500 Subject: [PATCH 018/102] =?UTF-8?q?Give=20error=20when=20git=20ingestion?= =?UTF-8?q?=20doesn=E2=80=99t=20use=20sha1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libstore/store-api.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f880cd5034c..c0303de679b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -178,6 +178,9 @@ StorePath Store::makeFixedOutputPath( const StorePathSet & references, bool hasSelfReference) const { + if (method == FileIngestionMethod::Git && hash.type != htSHA1) + throw Error("Git file ingestion must use sha1 hash"); + if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { From 6f7fab93580dc093d24285205fd7a969c8962940 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 12:24:05 -0500 Subject: [PATCH 019/102] Use SHA1 for Git objects This is really bad and dangerous! But Git migration to sha256 is still a ways away: https://lwn.net/Articles/811068/ So we need to allow it for the time being. --- src/libstore/daemon.cc | 11 ++++++++++- src/libstore/local-store.cc | 6 ++++-- src/nix/add-to-store.cc | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index aa2cbe6d487..7947e3f9708 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -708,11 +708,20 @@ static void performOp(TunnelLogger * logger, ref store, auto deriver = readString(from); if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash(readString(from), htSHA256); + + auto narHashString = readString(from); + info.references = readStorePaths(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); from >> info.ca >> repair >> dontCheckSigs; + + // git hashes are still using sha1 + if (hasPrefix(info.ca, "fixed:git:sha1:")) + info.narHash = Hash(narHashString, htSHA1); + else + info.narHash = Hash(narHashString, htSHA256); + if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e214c334b73..b4a66b445ad 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -909,7 +909,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash.type == htSHA256 || (i.narHash.type == htSHA1 && hasPrefix(i.ca, "fixed:git:"))); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1006,7 +1006,9 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr hashSink; - if (info.ca == "" || !info.references.count(info.path)) + if (hasPrefix(info.ca, "fixed:git:sha1:")) + hashSink = std::make_unique(htSHA1); // git objects use sha1 + else if (info.ca == "" || !info.references.count(info.path)) hashSink = std::make_unique(htSHA256); else hashSink = std::make_unique(htSHA256, storePathToHash(printStorePath(info.path))); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index c5b7c0368f7..efa223d9799 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -56,7 +56,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand else dumpPath(path, sink); - auto narHash = hashString(htSHA256, *sink.s); + auto narHash = hashString(git ? htSHA1 : htSHA256, *sink.s); ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart)); info.narHash = narHash; From ee989c62b726b7edaa9a161cf421078b037c7eca Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 12:27:48 -0500 Subject: [PATCH 020/102] Pass storeDir to restoreGit We need access to other things in the store. This is kind of dangerous though if things are added in the wrong order. --- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 6 +++--- src/libutil/git.cc | 14 ++++++-------- src/libutil/git.hh | 4 ++-- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 7947e3f9708..6fd582624eb 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -385,7 +385,7 @@ static void performOp(TunnelLogger * logger, ref store, } case FileIngestionMethod::Git: { ParseSink sink; - parseGit(sink, savedNAR); + parseGit(sink, savedNAR, store->storeDir); break; } case FileIngestionMethod::Flat: { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b4a66b445ad..6f5a3517e5f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1020,8 +1020,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, }); if (hasPrefix(info.ca, "fixed:git:")) - restoreGit(realPath, wrapperSource); - else + restoreGit(realPath, wrapperSource, realStoreDir); + else restorePath(realPath, wrapperSource); auto hashResult = hashSink->finish(); @@ -1084,7 +1084,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam } case FileIngestionMethod::Git: { StringSource source(dump); - restoreGit(realPath, source); + restoreGit(realPath, source, realStoreDir); break; } } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index cf9b222ec80..5581c4f0a7c 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -21,19 +21,17 @@ using namespace std::string_literals; namespace nix { -static void parse(ParseSink & sink, Source & source, const Path & path); +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir); // Converts a Path to a ParseSink -void restoreGit(const Path & path, Source & source) { - +void restoreGit(const Path & path, Source & source, const Path & storeDir) { RestoreSink sink; sink.dstPath = path; - parseGit(sink, source); - + parseGit(sink, source, storeDir); } -void parseGit(ParseSink & sink, Source & source) { - parse(sink, source, ""); +void parseGit(ParseSink & sink, Source & source, const Path & storeDir) { + parse(sink, source, "", storeDir); } string getStringUntil(Source & source, char byte) { @@ -53,7 +51,7 @@ string getString(Source & source, int n){ return std::string(v.begin(), v.end()); } -static void parse(ParseSink & sink, Source & source, const Path & path) { +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir) { auto type = getString(source, 5); if (type == "blob ") { diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 7b11f72be29..aed3de28d5e 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -12,9 +12,9 @@ enum struct GitMode { Regular, }; -void restoreGit(const Path & path, Source & source); +void restoreGit(const Path & path, Source & source, const Path & storeDir); -void parseGit(ParseSink & sink, Source & source); +void parseGit(ParseSink & sink, Source & source, const Path & storeDir); // Dumps a single file to a sink GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink); From 9cd3ac727028a7a44b2396e3e20abc8dceff2352 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 12:57:30 -0500 Subject: [PATCH 021/102] Pad 0s for dumpGitInternal --- src/libutil/git.cc | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 5581c4f0a7c..febbb12c101 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -127,7 +127,7 @@ GitMode dumpGitTree(const GitTree & entries, Sink & sink) case GitMode::Executable: mode = 100755; break; case GitMode::Regular: mode = 100644; break; } - s1 += (format("%6d %s\0%s"s) % mode % i.first % i.second.second.hash).str(); + s1 += (format("%06d %s\0%s"s) % mode % i.first % i.second.second.hash).str(); } std::string s2 = (format("tree %d\0%s"s) % s1.size() % s1).str(); @@ -138,39 +138,45 @@ GitMode dumpGitTree(const GitTree & entries, Sink & sink) return GitMode::Directory; } -// Returns the perm in addition -std::pair dumpGitHashInternal( - std::function()> makeHashSink, - const Path & path, PathFilter & filter) +static std::pair dumpGitHashInternal(HashType ht, const Path & path, PathFilter & filter); + +static GitMode dumpGitInternal(HashType ht, const Path & path, Sink & sink, PathFilter & filter) { - auto hashSink = makeHashSink(); struct stat st; GitMode perm; if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path '%1%'") % path); - if (S_ISREG(st.st_mode)) { - perm = dumpGitBlob(path, st, *hashSink); - } else if (S_ISDIR(st.st_mode)) { + if (S_ISREG(st.st_mode)) + perm = dumpGitBlob(path, st, sink); + else if (S_ISDIR(st.st_mode)) { GitTree entries; for (auto & i : readDirectory(path)) - if (filter(path + "/" + i.name)) { - entries[i.name] = dumpGitHashInternal(makeHashSink, path + "/" + i.name, filter); - } - perm = dumpGitTree(entries, *hashSink); - } else { - throw Error(format("file '%1%' has an unsupported type") % path); - } + if (filter(path + "/" + i.name)) + entries[i.name] = dumpGitHashInternal(ht, path + "/" + i.name, filter); + perm = dumpGitTree(entries, sink); + } else throw Error(format("file '%1%' has an unsupported type") % path); + + return perm; +} + +static std::pair dumpGitHashInternal(HashType ht, const Path & path, PathFilter & filter) +{ + auto hashSink = new HashSink(ht); + auto perm = dumpGitInternal(ht, path, *hashSink, filter); auto hash = hashSink->finish().first; return std::pair { perm, hash }; } -Hash dumpGitHash( - std::function()> makeHashSink, - const Path & path, PathFilter & filter) +Hash dumpGitHash(HashType ht, const Path & path, PathFilter & filter) +{ + return dumpGitHashInternal(ht, path, filter).second; +} + +void dumpGit(HashType ht, const Path & path, Sink & sink, PathFilter & filter) { - return dumpGitHashInternal(makeHashSink, path, filter).second; + dumpGitInternal(ht, path, sink, filter); } } From b5ed6a9c8b05c8c0af78c2d907a668d0571b3b89 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 13:39:00 -0500 Subject: [PATCH 022/102] Add createExecutableFile primitive to ParseSink This is needed to create files based on git permissions --- src/libutil/fs-sink.hh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index d3d1e43b79f..5ac49d83d41 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -13,6 +13,7 @@ struct ParseSink virtual void createDirectory(const Path & path) { }; virtual void createRegularFile(const Path & path) { }; + virtual void createExecutableFile(const Path & path) { }; virtual void isExecutable() { }; virtual void preallocateContents(unsigned long long size) { }; virtual void receiveContents(unsigned char * data, unsigned int len) { }; @@ -39,6 +40,13 @@ struct RestoreSink : ParseSink if (!fd) throw SysError(format("creating file '%1%'") % p); } + void createExecutableFile(const Path & path) + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0777); + if (!fd) throw SysError(format("creating file '%1%'") % p); + } + void isExecutable() { struct stat st; From 6ceba63920ab7c2057764afb9bebcd6b3bd7268a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 14:06:02 -0500 Subject: [PATCH 023/102] Read related git objects from nix store in parseGit --- src/libutil/git.cc | 53 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index febbb12c101..726248cce97 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -87,8 +87,6 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa if (perm != 100644 && perm != 100755 && perm != 644 && perm != 755 && perm != 40000) throw Error(format("Unknown Git permission: %d") % perm); - // TODO: handle permissions somehow - string name = getStringUntil(source, 0); left -= name.size(); left -= 1; @@ -96,10 +94,53 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa string hashs = getString(source, 20); left -= 20; - Hash hash(htSHA1); - std::copy(hashs.begin(), hashs.end(), hash.hash); - - sink.createSymlink(path + "/" + name, "../" + name); + Hash hash1(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash1.hash); + + Hash hash2 = hashString(htSHA256, "fixed:out:git:" + hash1.to_string(Base16) + ":"); + Hash hash3 = hashString(htSHA256, "output:out:" + hash2.to_string(Base16) + ":" + storeDir + ":" + name); + Hash hash4 = compressHash(hash3, 20); + + string entryName = hash4.to_string(Base32, false) + "-" + name; + Path entry = storeDir + "/" + entryName; + + struct stat st; + if (lstat(entry.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % entry); + + if (S_ISREG(st.st_mode)) { + if (perm == 40000) + throw SysError(format("file is a file but expected to be a directory '%1%'") % entry); + + if (perm == 100755 || perm == 755) + sink.createExecutableFile(path + "/" + name); + else + sink.createRegularFile(path + "/" + name); + + unsigned long long size = st.st_size; + sink.preallocateContents(size); + + unsigned long long left = size; + std::vector buf(65536); + + StringSink ssink; + readFile(entry, ssink); + AutoCloseFD fd = open(entry.c_str(), O_RDONLY | O_CLOEXEC); + + while (left) { + checkInterrupt(); + auto n = buf.size(); + if ((unsigned long long)n > left) n = left; + ssink(buf.data(), n); + sink.receiveContents(buf.data(), n); + left -= n; + } + } else if (S_ISDIR(st.st_mode)) { + if (perm != 40000) + throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); + + sink.createSymlink(path + "/" + name, "../" + entryName); + } else throw Error(format("file '%1%' has an unsupported type") % entry); } } else throw Error("input doesn't look like a Git object"); } From e44956d8298dba7a1656408d2b454cd94822b9d1 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 15:07:34 -0500 Subject: [PATCH 024/102] Allow sha1 in --hash results This updates the remote protocol to try to handle sha1 hashes. --- src/libstore/remote-store.cc | 7 ++++++- src/nix-store/nix-store.cc | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 842c2e17666..16408bb5907 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -375,13 +375,18 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, info = std::make_shared(path.clone()); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->narHash = Hash(readString(conn->from), htSHA256); + + auto narHashString = readString(conn->from); + info->references = readStorePaths(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); conn->from >> info->ca; + info->narHash = Hash(narHashString, hasPrefix(info->ca, "fixed:git:") ? htSHA1 : htSHA256); + } else { + info->narHash = Hash(narHashString, htSHA256); } } callback(std::move(info)); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 68fe6f72735..5b4ac6a5e71 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -372,7 +372,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); + assert(info->narHash.type == htSHA256 || (info->narHash.type == htSHA1 && hasPrefix(info->ca, "fixed:git:"))); cout << fmt("%s\n", info->narHash.to_string(Base32)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); From 646862eeebe0c98820f9d21c08a0c1e4506d05f2 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 15:08:09 -0500 Subject: [PATCH 025/102] Add test for adding git tree hash --- tests/add.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/add.sh b/tests/add.sh index 5ae4b1886e2..f9fe2475a7f 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -30,3 +30,10 @@ test "$hash1" = "sha256:$hash2" path5=$(nix add-to-store --git ./dummy) hash3=$(nix-store -q --hash $path5) test "$hash3" = "sha256:$(nix hash-git --base32 ./dummy)" + +mkdir -p dummy2 +echo hello > dummy2/hello +nix add-to-store --git ./dummy2/hello +path6=$(nix add-to-store --git ./dummy2) +hash4=$(nix-store -q --hash $path6) +test "$hash4" = "sha256:$(nix hash-git --base32 ./dummy2)" From 592e92664d670db7b7def000b544654e1893b47b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 15:25:49 -0500 Subject: [PATCH 026/102] Expand git tests --- tests/add.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/add.sh b/tests/add.sh index f9fe2475a7f..0c20f0dfff8 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -27,13 +27,26 @@ echo $hash2 test "$hash1" = "sha256:$hash2" +# Git tests + path5=$(nix add-to-store --git ./dummy) hash3=$(nix-store -q --hash $path5) -test "$hash3" = "sha256:$(nix hash-git --base32 ./dummy)" +test "$hash3" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy)" +rm -rf dummy2 mkdir -p dummy2 echo hello > dummy2/hello nix add-to-store --git ./dummy2/hello path6=$(nix add-to-store --git ./dummy2) hash4=$(nix-store -q --hash $path6) -test "$hash4" = "sha256:$(nix hash-git --base32 ./dummy2)" +test "$hash4" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy2)" + +rm -rf dummy3 +mkdir -p dummy3 +mkdir -p dummy3/hello +echo hello > dummy3/hello/hello +nix add-to-store --git ./dummy3/hello/hello +nix add-to-store --git ./dummy3/hello +path7=$(nix add-to-store --git ./dummy3) +hash5=$(nix-store -q --hash $path7) +test "$hash5" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy3)" From a4a038fd8a3fd8765f0b3b670bb64658de910006 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 17:14:40 -0500 Subject: [PATCH 027/102] Cleanup git.cc code --- src/libutil/git.cc | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 726248cce97..ef3fdfdd8ea 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -30,11 +30,13 @@ void restoreGit(const Path & path, Source & source, const Path & storeDir) { parseGit(sink, source, storeDir); } -void parseGit(ParseSink & sink, Source & source, const Path & storeDir) { +void parseGit(ParseSink & sink, Source & source, const Path & storeDir) +{ parse(sink, source, "", storeDir); } -string getStringUntil(Source & source, char byte) { +static string getStringUntil(Source & source, char byte) +{ string s; unsigned char n[1]; source(n, 1); @@ -45,13 +47,25 @@ string getStringUntil(Source & source, char byte) { return s; } -string getString(Source & source, int n){ +static string getString(Source & source, int n) +{ std::vector v(n); source(v.data(), n); return std::string(v.begin(), v.end()); } -static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir) { +// Unfortunately, no access to libstore headers here. +static Path getStorePath(const Path & storeDir, Hash hash, string name) +{ + Hash hash1 = hashString(htSHA256, "fixed:out:git:" + hash.to_string(Base16) + ":"); + Hash hash2 = hashString(htSHA256, "output:out:" + hash1.to_string(Base16) + ":" + storeDir + ":" + name); + Hash hash3 = compressHash(hash2, 20); + + return storeDir + "/" + hash3.to_string(Base32, false) + "-" + name; +} + +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir) +{ auto type = getString(source, 5); if (type == "blob ") { @@ -94,15 +108,10 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa string hashs = getString(source, 20); left -= 20; - Hash hash1(htSHA1); - std::copy(hashs.begin(), hashs.end(), hash1.hash); - - Hash hash2 = hashString(htSHA256, "fixed:out:git:" + hash1.to_string(Base16) + ":"); - Hash hash3 = hashString(htSHA256, "output:out:" + hash2.to_string(Base16) + ":" + storeDir + ":" + name); - Hash hash4 = compressHash(hash3, 20); + Hash hash(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); - string entryName = hash4.to_string(Base32, false) + "-" + name; - Path entry = storeDir + "/" + entryName; + Path entry = getStorePath(storeDir, hash, name); struct stat st; if (lstat(entry.c_str(), &st)) @@ -139,7 +148,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa if (perm != 40000) throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); - sink.createSymlink(path + "/" + name, "../" + entryName); + sink.createSymlink(path + "/" + name, entry); } else throw Error(format("file '%1%' has an unsupported type") % entry); } } else throw Error("input doesn't look like a Git object"); From 4ecb49d61c11fc4ea0f67a5a47dc641f58d1f3dd Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 18:02:08 -0500 Subject: [PATCH 028/102] Use path-based addToStore method for add-to-store This is more direct. --- src/nix/add-to-store.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index efa223d9799..c224d9b2223 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -63,10 +63,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(ingestionMethod, info.narHash); - if (!dryRun) { - auto source = StringSource { *sink.s }; - store->addToStore(info, source); - } + if (!dryRun) + store->addToStore(*namePart, path, ingestionMethod, git ? htSHA1 : htSHA256); logger->stdout("%s", store->printStorePath(info.path)); } From d68b774cc8fd1fadab76dc5b5eb621194cf9a627 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 18:26:18 -0500 Subject: [PATCH 029/102] Properly handle realStoreDir vs. storeDir in git.cc We need both to properly mess with the file system. storeDir goes into the hash while realStoreDir is what we read & write to. --- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/libutil/git.cc | 18 +++++++++--------- src/libutil/git.hh | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6fd582624eb..7270ae6cbc6 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -385,7 +385,7 @@ static void performOp(TunnelLogger * logger, ref store, } case FileIngestionMethod::Git: { ParseSink sink; - parseGit(sink, savedNAR, store->storeDir); + parseGit(sink, savedNAR, store->storeDir, store->storeDir); break; } case FileIngestionMethod::Flat: { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6f5a3517e5f..2c624d6460d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1020,7 +1020,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, }); if (hasPrefix(info.ca, "fixed:git:")) - restoreGit(realPath, wrapperSource, realStoreDir); + restoreGit(realPath, wrapperSource, realStoreDir, storeDir); else restorePath(realPath, wrapperSource); @@ -1084,7 +1084,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam } case FileIngestionMethod::Git: { StringSource source(dump); - restoreGit(realPath, source, realStoreDir); + restoreGit(realPath, source, realStoreDir, storeDir); break; } } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index ef3fdfdd8ea..b876962a43c 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -21,18 +21,18 @@ using namespace std::string_literals; namespace nix { -static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir); +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir); // Converts a Path to a ParseSink -void restoreGit(const Path & path, Source & source, const Path & storeDir) { +void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir) { RestoreSink sink; sink.dstPath = path; - parseGit(sink, source, storeDir); + parseGit(sink, source, realStoreDir, storeDir); } -void parseGit(ParseSink & sink, Source & source, const Path & storeDir) +void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir) { - parse(sink, source, "", storeDir); + parse(sink, source, "", realStoreDir, storeDir); } static string getStringUntil(Source & source, char byte) @@ -55,16 +55,16 @@ static string getString(Source & source, int n) } // Unfortunately, no access to libstore headers here. -static Path getStorePath(const Path & storeDir, Hash hash, string name) +static Path getStorePath(const Path & realStoreDir, const Path & storeDir, Hash hash, string name) { Hash hash1 = hashString(htSHA256, "fixed:out:git:" + hash.to_string(Base16) + ":"); Hash hash2 = hashString(htSHA256, "output:out:" + hash1.to_string(Base16) + ":" + storeDir + ":" + name); Hash hash3 = compressHash(hash2, 20); - return storeDir + "/" + hash3.to_string(Base32, false) + "-" + name; + return realStoreDir + "/" + hash3.to_string(Base32, false) + "-" + name; } -static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir) +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir) { auto type = getString(source, 5); @@ -111,7 +111,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa Hash hash(htSHA1); std::copy(hashs.begin(), hashs.end(), hash.hash); - Path entry = getStorePath(storeDir, hash, name); + Path entry = getStorePath(realStoreDir, storeDir, hash, name); struct stat st; if (lstat(entry.c_str(), &st)) diff --git a/src/libutil/git.hh b/src/libutil/git.hh index aed3de28d5e..1d773d3750f 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -12,9 +12,9 @@ enum struct GitMode { Regular, }; -void restoreGit(const Path & path, Source & source, const Path & storeDir); +void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir); -void parseGit(ParseSink & sink, Source & source, const Path & storeDir); +void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir); // Dumps a single file to a sink GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink); From 18d4a3d5f37a7f24928a7a0dfd6abb92eec2c95d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 18:52:31 -0500 Subject: [PATCH 030/102] Allow sending git objects over daemon --- src/libstore/daemon.cc | 2 +- src/libstore/remote-store.cc | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 7270ae6cbc6..4bac5bff68b 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -398,7 +398,7 @@ static void performOp(TunnelLogger * logger, ref store, if (!savedRegular.regular) throw Error("regular file expected"); auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, + method == FileIngestionMethod::Flat ? savedRegular.s : *savedNAR.data, baseName, method, hashAlgo); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 16408bb5907..019d3e6f02d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -8,6 +8,7 @@ #include "derivations.hh" #include "pool.hh" #include "finally.hh" +#include "git.hh" #include #include @@ -493,7 +494,6 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - if (method == FileIngestionMethod::Git) throw Error("cannot remotely add to store using the git file ingestion method"); auto conn(getConnection()); @@ -512,7 +512,10 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - dumpPath(srcPath, conn->to, filter); + if (method == FileIngestionMethod::Git) + dumpGit(hashAlgo, srcPath, conn->to, filter); + else + dumpPath(srcPath, conn->to, filter); } conn->to.warn = false; conn.processStderr(); From d8f34fd166b60848b7614db1e8f8b69799b7947c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 18:52:50 -0500 Subject: [PATCH 031/102] Recursively add to store from git ingestion --- src/libstore/local-store.cc | 11 ++++++++++- src/libstore/remote-store.cc | 13 +++++++++++-- tests/add.sh | 3 --- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2c624d6460d..13c7109c31f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1131,7 +1131,16 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, case FileIngestionMethod::Recursive: dumpPath(srcPath, sink, filter); break; - case FileIngestionMethod::Git:{ + } + case FileIngestionMethod::Git: { + // recursively add to store if path is a directory + struct stat st; + if (lstat(srcPath.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % srcPath); + if (S_ISDIR(st.st_mode)) + for (auto & i : readDirectory(srcPath)) + addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair); + dumpGit(srcPath, sink, filter); break; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 019d3e6f02d..b814ed67865 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -494,10 +494,19 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + Path srcPath(absPath(_srcPath)); - auto conn(getConnection()); + // recursively add to store if path is a directory + if (method == FileIngestionMethod::Git) { + struct stat st; + if (lstat(srcPath.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % srcPath); + if (S_ISDIR(st.st_mode)) + for (auto & i : readDirectory(srcPath)) + addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair); + } - Path srcPath(absPath(_srcPath)); + auto conn(getConnection()); conn->to << wopAddToStore diff --git a/tests/add.sh b/tests/add.sh index 0c20f0dfff8..389bdb57245 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -36,7 +36,6 @@ test "$hash3" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy)" rm -rf dummy2 mkdir -p dummy2 echo hello > dummy2/hello -nix add-to-store --git ./dummy2/hello path6=$(nix add-to-store --git ./dummy2) hash4=$(nix-store -q --hash $path6) test "$hash4" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy2)" @@ -45,8 +44,6 @@ rm -rf dummy3 mkdir -p dummy3 mkdir -p dummy3/hello echo hello > dummy3/hello/hello -nix add-to-store --git ./dummy3/hello/hello -nix add-to-store --git ./dummy3/hello path7=$(nix add-to-store --git ./dummy3) hash5=$(nix-store -q --hash $path7) test "$hash5" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy3)" From 27ffbc7c1425c1091db8fb73e37e2e1e2c3495fa Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 23:02:17 -0500 Subject: [PATCH 032/102] Add hash to local store correctly from dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Need to use the htSHA1 we were given, don’t recompute. --- src/libstore/local-store.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 13c7109c31f..5e6b4b94092 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1096,7 +1096,12 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (method == FileIngestionMethod::Recursive) { + if (method == FileIngestionMethod::Git) { + if (hashAlgo != htSHA1) + throw Error("Git must use sha1 hash"); + hash.first = h; + hash.second = dump.size(); + } else if (method == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else From 05cbfdef9f76cafd5093dd08a8bfdea85f5bb297 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 23:07:50 -0500 Subject: [PATCH 033/102] Move git tests to own file this makes maintanence easier --- tests/add.sh | 21 --------------------- tests/git.sh | 36 ++++++++++++++++++++++++++++++++++++ tests/hash.sh | 15 --------------- tests/local.mk | 3 ++- 4 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 tests/git.sh diff --git a/tests/add.sh b/tests/add.sh index 389bdb57245..e26e05843d7 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -26,24 +26,3 @@ hash2=$(nix-hash --type sha256 --base32 ./dummy) echo $hash2 test "$hash1" = "sha256:$hash2" - -# Git tests - -path5=$(nix add-to-store --git ./dummy) -hash3=$(nix-store -q --hash $path5) -test "$hash3" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy)" - -rm -rf dummy2 -mkdir -p dummy2 -echo hello > dummy2/hello -path6=$(nix add-to-store --git ./dummy2) -hash4=$(nix-store -q --hash $path6) -test "$hash4" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy2)" - -rm -rf dummy3 -mkdir -p dummy3 -mkdir -p dummy3/hello -echo hello > dummy3/hello/hello -path7=$(nix add-to-store --git ./dummy3) -hash5=$(nix-store -q --hash $path7) -test "$hash5" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy3)" diff --git a/tests/git.sh b/tests/git.sh new file mode 100644 index 00000000000..08ec310e5af --- /dev/null +++ b/tests/git.sh @@ -0,0 +1,36 @@ +source common.sh + +try () { + hash=$(nix hash-git --base16 --type sha1 $TEST_ROOT/hash-path) + if test "$hash" != "$1"; then + echo "git hash, expected $1, got $hash" + exit 1 + fi +} + +rm -rf $TEST_ROOT/hash-path +mkdir $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path/hello + +try "117c62a8c5e01758bd284126a6af69deab9dbbe2" + +rm -rf $TEST_ROOT/dummy1 +echo Hello World! > $TEST_ROOT/dummy1 +path1=$(nix add-to-store --git $TEST_ROOT/dummy1) +hash1=$(nix-store -q --hash $path1) +test "$hash1" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy1)" + +rm -rf $TEST_ROOT/dummy2 +mkdir -p $TEST_ROOT/dummy2 +echo hello > $TEST_ROOT/dummy2/hello +path2=$(nix add-to-store --git $TEST_ROOT/dummy2) +hash2=$(nix-store -q --hash $path2) +test "$hash2" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy2)" + +rm -rf $TEST_ROOT/dummy3 +mkdir -p $TEST_ROOT/dummy3 +mkdir -p $TEST_ROOT/dummy3/hello +echo hello > $TEST_ROOT/dummy3/hello/hello +path3=$(nix add-to-store --git $TEST_ROOT/dummy3) +hash3=$(nix-store -q --hash $path3) +test "$hash3" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy3)" diff --git a/tests/hash.sh b/tests/hash.sh index 33b5bef7595..4cfc9790101 100644 --- a/tests/hash.sh +++ b/tests/hash.sh @@ -85,18 +85,3 @@ try3() { try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" - -# Git. -try4 () { - hash=$(nix hash-git --base16 --type sha1 $TEST_ROOT/hash-path) - if test "$hash" != "$1"; then - echo "git hash, expected $1, got $hash" - exit 1 - fi -} - -rm -rf $TEST_ROOT/hash-path -mkdir $TEST_ROOT/hash-path -echo "Hello World" > $TEST_ROOT/hash-path/hello - -try4 "117c62a8c5e01758bd284126a6af69deab9dbbe2" diff --git a/tests/local.mk b/tests/local.mk index 56e5640ca33..35a80a16a5a 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -30,7 +30,8 @@ nix_tests = \ nix-copy-ssh.sh \ post-hook.sh \ function-trace.sh \ - recursive.sh + recursive.sh \ + git.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) From 97ce2cc5d3d83d33f6056d125f1a41ca46af07d4 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 23:53:02 -0500 Subject: [PATCH 034/102] Revert "Depend on install for installcheck" This reverts commit 7b6186dc661c036bb251ba3158622120acaebc3f. --- mk/tests.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/tests.mk b/mk/tests.mk index c3c1f4c28dd..70c30661b95 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -8,7 +8,7 @@ define run-install-test endef # Color code from https://unix.stackexchange.com/a/10065 -installcheck: install +installcheck: @total=0; failed=0; \ red=""; \ green=""; \ From 229ce9c454cf90344ebdecc3d43219395fdfe9cf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 10:28:09 -0500 Subject: [PATCH 035/102] Make computeStorePathForPath work in Git ingestion --- src/libstore/store-api.cc | 18 +++++++++++++++--- src/libutil/hash.cc | 8 ++++++++ src/libutil/hash.hh | 3 +++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c0303de679b..4f99cfe2fa4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -209,9 +209,21 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, std::pair Store::computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const { - Hash h = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath, filter).first - : hashFile(hashAlgo, srcPath); + Hash h; + switch (method) { + case FileIngestionMethod::Recursive: { + h = hashPath(hashAlgo, srcPath, filter).first; + break; + } + case FileIngestionMethod::Git: { + h = hashGit(hashAlgo, srcPath, filter).first; + break; + } + case FileIngestionMethod::Flat: { + h = hashFile(hashAlgo, srcPath); + break; + } + } return std::make_pair(makeFixedOutputPath(method, h, name), h); } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 7caee1da7f8..b3f3db23178 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -6,6 +6,7 @@ #include "hash.hh" #include "archive.hh" +#include "git.hh" #include "util.hh" #include "istringstream_nocopy.hh" @@ -307,6 +308,13 @@ HashResult hashPath( return sink.finish(); } +HashResult hashGit( + HashType ht, const Path & path, PathFilter & filter) +{ + HashSink sink(ht); + dumpGit(ht, path, sink, filter); + return sink.finish(); +} Hash compressHash(const Hash & hash, unsigned int newSize) { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ea9fca3e74f..1af94c2b47c 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -110,6 +110,9 @@ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); +HashResult hashGit(HashType ht, const Path & path, + PathFilter & filter = defaultPathFilter); + /* Compress a hash to the specified number of bytes by cyclically XORing bytes together. */ Hash compressHash(const Hash & hash, unsigned int newSize); From 031fa72c20c2c5cd4ed996025df1ad44ea134b57 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 11:05:09 -0500 Subject: [PATCH 036/102] Use SHA256 for narHash narHash is the hash of the nar, not the git objects. --- src/libstore/daemon.cc | 11 +---------- src/libstore/local-store.cc | 17 +++++------------ src/libstore/remote-store.cc | 9 ++------- src/libstore/store-api.cc | 4 +++- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 8 ++++---- tests/git.sh | 6 +++--- 7 files changed, 19 insertions(+), 38 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4bac5bff68b..117ee8c3585 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -708,20 +708,11 @@ static void performOp(TunnelLogger * logger, ref store, auto deriver = readString(from); if (deriver != "") info.deriver = store->parseStorePath(deriver); - - auto narHashString = readString(from); - + info.narHash = Hash(readString(from), htSHA256); info.references = readStorePaths(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); from >> info.ca >> repair >> dontCheckSigs; - - // git hashes are still using sha1 - if (hasPrefix(info.ca, "fixed:git:sha1:")) - info.narHash = Hash(narHashString, htSHA1); - else - info.narHash = Hash(narHashString, htSHA256); - if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5e6b4b94092..031ef367cae 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -909,7 +909,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256 || (i.narHash.type == htSHA1 && hasPrefix(i.ca, "fixed:git:"))); + assert(i.narHash.type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1006,9 +1006,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr hashSink; - if (hasPrefix(info.ca, "fixed:git:sha1:")) - hashSink = std::make_unique(htSHA1); // git objects use sha1 - else if (info.ca == "" || !info.references.count(info.path)) + if (info.ca == "" || !info.references.count(info.path)) hashSink = std::make_unique(htSHA256); else hashSink = std::make_unique(htSHA256, storePathToHash(printStorePath(info.path))); @@ -1051,7 +1049,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { - Hash h = hashString(hashAlgo, dump); + Hash h = hashString(method == FileIngestionMethod::Git ? htSHA1 : hashAlgo, dump); auto dstPath = makeFixedOutputPath(method, h, name); @@ -1096,12 +1094,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (method == FileIngestionMethod::Git) { - if (hashAlgo != htSHA1) - throw Error("Git must use sha1 hash"); - hash.first = h; - hash.second = dump.size(); - } else if (method == FileIngestionMethod::Recursive) { + if (method == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else @@ -1146,7 +1139,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, for (auto & i : readDirectory(srcPath)) addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair); - dumpGit(srcPath, sink, filter); + dumpGit(htSHA1, srcPath, sink, filter); break; } case FileIngestionMethod::Flat: { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b814ed67865..dbc70b5082b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -376,18 +376,13 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, info = std::make_shared(path.clone()); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - - auto narHashString = readString(conn->from); - + info->narHash = Hash(readString(conn->from), htSHA256); info->references = readStorePaths(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); conn->from >> info->ca; - info->narHash = Hash(narHashString, hasPrefix(info->ca, "fixed:git:") ? htSHA1 : htSHA256); - } else { - info->narHash = Hash(narHashString, htSHA256); } } callback(std::move(info)); @@ -522,7 +517,7 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { Finally cleanup([&]() { connections->decCapacity(); }); if (method == FileIngestionMethod::Git) - dumpGit(hashAlgo, srcPath, conn->to, filter); + dumpGit(htSHA1, srcPath, conn->to, filter); else dumpPath(srcPath, conn->to, filter); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4f99cfe2fa4..e94ed8a9ea0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -216,7 +216,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, break; } case FileIngestionMethod::Git: { - h = hashGit(hashAlgo, srcPath, filter).first; + h = hashGit(htSHA1, srcPath, filter).first; break; } case FileIngestionMethod::Flat: { @@ -853,6 +853,8 @@ Strings ValidPathInfo::shortRefs() const std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) { + if (method == FileIngestionMethod::Git && hash.type != htSHA1) + throw Error("git file ingestion must use sha1 hashes"); return "fixed:" + ingestionMethodPrefix(method) + hash.to_string(); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b4ac6a5e71..68fe6f72735 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -372,7 +372,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256 || (info->narHash.type == htSHA1 && hasPrefix(info->ca, "fixed:git:"))); + assert(info->narHash.type == htSHA256); cout << fmt("%s\n", info->narHash.to_string(Base32)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index c224d9b2223..731951bd0f7 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -56,12 +56,12 @@ struct CmdAddToStore : MixDryRun, StoreCommand else dumpPath(path, sink); - auto narHash = hashString(git ? htSHA1 : htSHA256, *sink.s); + auto hash = hashString(git ? htSHA1 : htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart)); - info.narHash = narHash; + ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); + info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(ingestionMethod, info.narHash); + info.ca = makeFixedOutputCA(ingestionMethod, hash); if (!dryRun) store->addToStore(*namePart, path, ingestionMethod, git ? htSHA1 : htSHA256); diff --git a/tests/git.sh b/tests/git.sh index 08ec310e5af..bd5a4742249 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -18,14 +18,14 @@ rm -rf $TEST_ROOT/dummy1 echo Hello World! > $TEST_ROOT/dummy1 path1=$(nix add-to-store --git $TEST_ROOT/dummy1) hash1=$(nix-store -q --hash $path1) -test "$hash1" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy1)" +test "$hash1" = "sha256:1brffhvj2c0z6x8qismd43m0iy8dsgfmy10bgg9w11szway2wp9v" rm -rf $TEST_ROOT/dummy2 mkdir -p $TEST_ROOT/dummy2 echo hello > $TEST_ROOT/dummy2/hello path2=$(nix add-to-store --git $TEST_ROOT/dummy2) hash2=$(nix-store -q --hash $path2) -test "$hash2" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy2)" +test "$hash2" = "sha256:1wnx6ldp06c1riiyhgqvfmhzpm664066i5hdq5fdcf1wg19mz45i" rm -rf $TEST_ROOT/dummy3 mkdir -p $TEST_ROOT/dummy3 @@ -33,4 +33,4 @@ mkdir -p $TEST_ROOT/dummy3/hello echo hello > $TEST_ROOT/dummy3/hello/hello path3=$(nix add-to-store --git $TEST_ROOT/dummy3) hash3=$(nix-store -q --hash $path3) -test "$hash3" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy3)" +test "$hash3" = "sha256:153908hlsd9bhcpqxsfrcvqwqmxbhrizzjzn4rnggh3blzzdzim4" From d49c87323654575c37ca9c522ccf88b1ab2b7930 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 13:39:53 -0500 Subject: [PATCH 037/102] Use correct narHash in add-to-store We need to dump path and dump git here so that the hashes match what is expected elsewhere. --- src/nix/add-to-store.cc | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 731951bd0f7..ced8704ef0a 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -51,15 +51,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto ingestionMethod = git ? FileIngestionMethod::Git : FileIngestionMethod::Recursive; StringSink sink; - if (git) - dumpGit(path, sink); - else - dumpPath(path, sink); + dumpPath(path, sink); - auto hash = hashString(git ? htSHA1 : htSHA256, *sink.s); + auto narHash = hashString(htSHA256, *sink.s); + auto hash = git ? dumpGitHash(htSHA1, path) : narHash; ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); - info.narHash = hashString(htSHA256, *sink.s); + info.narHash = narHash; info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(ingestionMethod, hash); From cd3ef3f6a9469528feed6010a2e20e14a039fd08 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 13:48:14 -0500 Subject: [PATCH 038/102] Throw error when hashAlgo != SHA1 on git ingestion This should always use sha1 hash type, but we want to make sure the caller knows that. So just throw an error instead of ignoring hashAlgo on Git ingestion. --- src/libstore/local-store.cc | 10 ++++++++-- src/libstore/remote-store.cc | 5 ++++- src/libstore/store-api.cc | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 031ef367cae..b742857ccbb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1049,7 +1049,10 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { - Hash h = hashString(method == FileIngestionMethod::Git ? htSHA1 : hashAlgo, dump); + if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) + throw Error("git ingestion must use sha1 hash"); + + Hash h = hashString(hashAlgo, dump); auto dstPath = makeFixedOutputPath(method, h, name); @@ -1121,6 +1124,9 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); + if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) + throw Error("git ingestion must use sha1 hash"); + /* Read the whole path into memory. This is not a very scalable method for very large paths, but `copyPath' is mainly used for small files. */ @@ -1139,7 +1145,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, for (auto & i : readDirectory(srcPath)) addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair); - dumpGit(htSHA1, srcPath, sink, filter); + dumpGit(hashAlgo, srcPath, sink, filter); break; } case FileIngestionMethod::Flat: { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index dbc70b5082b..fbc161cc92c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -489,6 +489,9 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) + throw Error("git ingestion must use sha1 hash"); + Path srcPath(absPath(_srcPath)); // recursively add to store if path is a directory @@ -517,7 +520,7 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { Finally cleanup([&]() { connections->decCapacity(); }); if (method == FileIngestionMethod::Git) - dumpGit(htSHA1, srcPath, conn->to, filter); + dumpGit(hashAlgo, srcPath, conn->to, filter); else dumpPath(srcPath, conn->to, filter); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e94ed8a9ea0..39c4009b5a0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -216,7 +216,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, break; } case FileIngestionMethod::Git: { - h = hashGit(htSHA1, srcPath, filter).first; + h = hashGit(hashAlgo, srcPath, filter).first; break; } case FileIngestionMethod::Flat: { From 4dae98e044294f363ce9b97afeab8914e2573151 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 14:02:53 -0500 Subject: [PATCH 039/102] =?UTF-8?q?Don=E2=80=99t=20include=20realStoreDir?= =?UTF-8?q?=20refs=20in=20git=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit symlinks should be relative so that they look like: ../s5c0hnz9qfnpnn1bszfxicgz21d1fam3-dummy3 instead of /build/nix-test/store/s5c0hnz9qfnpnn1bszfxicgz21d1fam3-dummy3 This way our hashes will work with any real store dir. Note that /nix/store is still embedded in the store entry function. --- src/libutil/git.cc | 9 +++++---- tests/git.sh | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index b876962a43c..3472e812cce 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -55,13 +55,13 @@ static string getString(Source & source, int n) } // Unfortunately, no access to libstore headers here. -static Path getStorePath(const Path & realStoreDir, const Path & storeDir, Hash hash, string name) +static string getStoreEntry(const Path & storeDir, Hash hash, string name) { Hash hash1 = hashString(htSHA256, "fixed:out:git:" + hash.to_string(Base16) + ":"); Hash hash2 = hashString(htSHA256, "output:out:" + hash1.to_string(Base16) + ":" + storeDir + ":" + name); Hash hash3 = compressHash(hash2, 20); - return realStoreDir + "/" + hash3.to_string(Base32, false) + "-" + name; + return hash3.to_string(Base32, false) + "-" + name; } static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir) @@ -111,7 +111,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa Hash hash(htSHA1); std::copy(hashs.begin(), hashs.end(), hash.hash); - Path entry = getStorePath(realStoreDir, storeDir, hash, name); + string entryName = getStoreEntry(storeDir, hash, name); + Path entry = realStoreDir + "/" + entryName; struct stat st; if (lstat(entry.c_str(), &st)) @@ -148,7 +149,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa if (perm != 40000) throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); - sink.createSymlink(path + "/" + name, entry); + sink.createSymlink(path + "/" + name, "../" + entryName); } else throw Error(format("file '%1%' has an unsupported type") % entry); } } else throw Error("input doesn't look like a Git object"); diff --git a/tests/git.sh b/tests/git.sh index bd5a4742249..d9daa9f2d77 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -33,4 +33,4 @@ mkdir -p $TEST_ROOT/dummy3/hello echo hello > $TEST_ROOT/dummy3/hello/hello path3=$(nix add-to-store --git $TEST_ROOT/dummy3) hash3=$(nix-store -q --hash $path3) -test "$hash3" = "sha256:153908hlsd9bhcpqxsfrcvqwqmxbhrizzjzn4rnggh3blzzdzim4" +test "$hash3" = "sha256:06psnck6rb12xyd8h4y2j801xf2lvhjwx9gxzxk25h922nm71lmf" From 1b14945a3bf741a44e520d156c01ca79a50975bb Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 15:16:30 -0500 Subject: [PATCH 040/102] Use FdSink to simplify copying --- src/libutil/fs-sink.hh | 7 +++++++ src/libutil/git.cc | 24 +++++------------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 5ac49d83d41..7d8764e8ba9 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -19,6 +19,8 @@ struct ParseSink virtual void receiveContents(unsigned char * data, unsigned int len) { }; virtual void createSymlink(const Path & path, const string & target) { }; + + virtual int getFD() { return 0; }; }; struct RestoreSink : ParseSink @@ -81,6 +83,11 @@ struct RestoreSink : ParseSink Path p = dstPath + path; nix::createSymlink(target, p); } + + int getFD() + { + return fd.get(); + } }; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 3472e812cce..f2be3eaad54 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -16,6 +16,7 @@ #include "hash.hh" #include "git.hh" +#include "serialise.hh" using namespace std::string_literals; @@ -112,7 +113,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa std::copy(hashs.begin(), hashs.end(), hash.hash); string entryName = getStoreEntry(storeDir, hash, name); - Path entry = realStoreDir + "/" + entryName; + Path entry = absPath(realStoreDir + "/" + entryName); struct stat st; if (lstat(entry.c_str(), &st)) @@ -127,24 +128,9 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa else sink.createRegularFile(path + "/" + name); - unsigned long long size = st.st_size; - sink.preallocateContents(size); - - unsigned long long left = size; - std::vector buf(65536); - - StringSink ssink; - readFile(entry, ssink); - AutoCloseFD fd = open(entry.c_str(), O_RDONLY | O_CLOEXEC); - - while (left) { - checkInterrupt(); - auto n = buf.size(); - if ((unsigned long long)n > left) n = left; - ssink(buf.data(), n); - sink.receiveContents(buf.data(), n); - left -= n; - } + sink.preallocateContents(st.st_size); + FdSink fsink(sink.getFD()); + readFile(entry, fsink); } else if (S_ISDIR(st.st_mode)) { if (perm != 40000) throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); From 54ee74e9b600c094bc2c69c5b67c502b30c88ee5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 15:17:09 -0500 Subject: [PATCH 041/102] Fixup bad hash in tests/git.sh --- tests/git.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/git.sh b/tests/git.sh index d9daa9f2d77..f8890ce50e2 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -25,7 +25,7 @@ mkdir -p $TEST_ROOT/dummy2 echo hello > $TEST_ROOT/dummy2/hello path2=$(nix add-to-store --git $TEST_ROOT/dummy2) hash2=$(nix-store -q --hash $path2) -test "$hash2" = "sha256:1wnx6ldp06c1riiyhgqvfmhzpm664066i5hdq5fdcf1wg19mz45i" +test "$hash2" = "sha256:1pgyz59p65wd11vfxp3vi673ijwjfg7i4ynlqlsgzvg9dvh67dpj" rm -rf $TEST_ROOT/dummy3 mkdir -p $TEST_ROOT/dummy3 From b1e1ace44934fc847abb46586396d41d6acfc66d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 15:32:27 -0500 Subject: [PATCH 042/102] Update tests/git.sh hashes --- tests/git.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/git.sh b/tests/git.sh index f8890ce50e2..afc39da295f 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -22,15 +22,15 @@ test "$hash1" = "sha256:1brffhvj2c0z6x8qismd43m0iy8dsgfmy10bgg9w11szway2wp9v" rm -rf $TEST_ROOT/dummy2 mkdir -p $TEST_ROOT/dummy2 -echo hello > $TEST_ROOT/dummy2/hello +echo Hello World! > $TEST_ROOT/dummy2/hello path2=$(nix add-to-store --git $TEST_ROOT/dummy2) hash2=$(nix-store -q --hash $path2) -test "$hash2" = "sha256:1pgyz59p65wd11vfxp3vi673ijwjfg7i4ynlqlsgzvg9dvh67dpj" +test "$hash2" = "sha256:1vhv7zxam7x277q0y0jcypm7hwhccbzss81vkdgf0ww5sm2am4y0" rm -rf $TEST_ROOT/dummy3 mkdir -p $TEST_ROOT/dummy3 mkdir -p $TEST_ROOT/dummy3/hello -echo hello > $TEST_ROOT/dummy3/hello/hello +echo Hello World! > $TEST_ROOT/dummy3/hello/hello path3=$(nix add-to-store --git $TEST_ROOT/dummy3) hash3=$(nix-store -q --hash $path3) -test "$hash3" = "sha256:06psnck6rb12xyd8h4y2j801xf2lvhjwx9gxzxk25h922nm71lmf" +test "$hash3" = "sha256:14qzv6n72xgr71sgb3ilpchaqzrh1vm61qj3qf0vmdfr721sm48z" From 7fe9a4891aa8c959390f87b52549ba9321bcf9fa Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 15:32:34 -0500 Subject: [PATCH 043/102] =?UTF-8?q?Don=E2=80=99t=20preallocate=20file=20si?= =?UTF-8?q?ze?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this shouldn’t be needed - FdSink handles this for us. --- src/libutil/git.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index f2be3eaad54..c881abb5a25 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -128,7 +128,6 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa else sink.createRegularFile(path + "/" + name); - sink.preallocateContents(st.st_size); FdSink fsink(sink.getFD()); readFile(entry, fsink); } else if (S_ISDIR(st.st_mode)) { From eb90cc65658b0bf2350e2500d547eb4847beb707 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 16:25:41 -0500 Subject: [PATCH 044/102] Copy instead of symlinking directories Symlinks are resolved in the nar format so we end up with references to the real store dir. Copying avoids this and gives us a stable hash. Also update the ParseSink api with two methods: - copyFile - copyDirectory both are needed to properly implement git parsing. --- src/libutil/fs-sink.hh | 15 ++++++++++++--- src/libutil/git.cc | 6 +++--- tests/git.sh | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 7d8764e8ba9..85e192e0743 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -1,6 +1,7 @@ #pragma once #include +#include #include "types.hh" #include "serialise.hh" @@ -20,7 +21,8 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; - virtual int getFD() { return 0; }; + virtual void copyFile(const Path & source) { }; + virtual void copyDirectory(const Path & source, const Path & destination) { }; }; struct RestoreSink : ParseSink @@ -84,9 +86,16 @@ struct RestoreSink : ParseSink nix::createSymlink(target, p); } - int getFD() + void copyFile(const Path & source) { - return fd.get(); + FdSink sink(fd.get()); + readFile(source, sink); + } + + void copyDirectory(const Path & source, const Path & destination) + { + Path p = dstPath + destination; + std::filesystem::copy(source, p); } }; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index c881abb5a25..aa96fd9bd99 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -128,13 +128,13 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa else sink.createRegularFile(path + "/" + name); - FdSink fsink(sink.getFD()); - readFile(entry, fsink); + sink.copyFile(entry); } else if (S_ISDIR(st.st_mode)) { if (perm != 40000) throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); - sink.createSymlink(path + "/" + name, "../" + entryName); + sink.createDirectory(path + "/" + name); + sink.copyDirectory(realStoreDir + "/" + entryName, path + "/" + name); } else throw Error(format("file '%1%' has an unsupported type") % entry); } } else throw Error("input doesn't look like a Git object"); diff --git a/tests/git.sh b/tests/git.sh index afc39da295f..e700dcc785c 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -33,4 +33,4 @@ mkdir -p $TEST_ROOT/dummy3/hello echo Hello World! > $TEST_ROOT/dummy3/hello/hello path3=$(nix add-to-store --git $TEST_ROOT/dummy3) hash3=$(nix-store -q --hash $path3) -test "$hash3" = "sha256:14qzv6n72xgr71sgb3ilpchaqzrh1vm61qj3qf0vmdfr721sm48z" +test "$hash3" = "sha256:1i2x80840igikhbyy7nqf08ymx3a6n83x1fzyrxvddf0sdl5nqvp" From b12025953072411dd8aedab5f0d487b504583743 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 18:29:50 -0500 Subject: [PATCH 045/102] Use custom copyDirectory instead of libc++fs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libc++fs isn’t alway available --- src/libutil/fs-sink.hh | 16 ++++++++++++++-- src/libutil/git.cc | 1 - 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 85e192e0743..efd13685df3 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -1,7 +1,6 @@ #pragma once #include -#include #include "types.hh" #include "serialise.hh" @@ -95,7 +94,20 @@ struct RestoreSink : ParseSink void copyDirectory(const Path & source, const Path & destination) { Path p = dstPath + destination; - std::filesystem::copy(source, p); + createDirectory(destination); + for (auto & i : readDirectory(source)) { + struct stat st; + Path entry = source + "/" + i.name; + if (lstat(entry.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % entry); + if (S_ISREG(st.st_mode)) { + createRegularFile(destination + "/" + i.name); + copyFile(entry); + } else if (S_ISDIR(st.st_mode)) + copyDirectory(entry, destination + "/" + i.name); + else + throw Error(format("Unknown file: %s") % entry); + } } }; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index aa96fd9bd99..3e613949a26 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -133,7 +133,6 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa if (perm != 40000) throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); - sink.createDirectory(path + "/" + name); sink.copyDirectory(realStoreDir + "/" + entryName, path + "/" + name); } else throw Error(format("file '%1%' has an unsupported type") % entry); } From 7432a9dca60b963c4f8cba422bdec6967f1a1863 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 19:06:55 -0500 Subject: [PATCH 046/102] Fixup bad rebase --- src/libstore/local-store.cc | 2 +- src/libutil/git.hh | 9 +++------ src/nix/hash.cc | 21 ++++++++------------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b742857ccbb..14c816e7282 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1132,7 +1132,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, small files. */ StringSink sink; switch (method) { - case FileIngestionMethod::Recursive: + case FileIngestionMethod::Recursive: { dumpPath(srcPath, sink, filter); break; } diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 1d773d3750f..06a56d55b70 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -25,11 +25,8 @@ typedef std::map> GitTree; GitMode dumpGitTree(const GitTree & entries, Sink & sink); // Recursively dumps path, hashing as we go -Hash dumpGitHash( - std::function()>, - const Path & path, - PathFilter & filter = defaultPathFilter); +Hash dumpGitHash(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); + +void dumpGit(HashType ht, const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); -// N.B. There is no way to recursively dump to a sink, as that doesn't make -// sense with the git hash/data model where the information is Merklized. } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 3a8c82da17b..41a66676967 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -57,34 +57,29 @@ struct CmdHash : Command { for (auto path : paths) { - auto makeHashSink = [&]() -> std::unique_ptr { - std::unique_ptr t; - if (modulus) - t = std::make_unique(ht, *modulus); - else - t = std::make_unique(ht); - return t; - }; + std::unique_ptr hashSink; + if (modulus) + hashSink = std::make_unique(ht, *modulus); + else + hashSink = std::make_unique(ht); Hash h; switch (mode) { case FileIngestionMethod::Flat: { - auto hashSink = makeHashSink(); readFile(path, *hashSink); - h = hashSink->finish().first; break; } case FileIngestionMethod::Recursive: { - auto hashSink = makeHashSink(); dumpPath(path, *hashSink); - h = hashSink->finish().first; break; } case FileIngestionMethod::Git: - h = dumpGitHash(makeHashSink, path); + dumpGit(ht, path, *hashSink); break; } + h = hashSink->finish().first; + if (truncate && h.hashSize > 20) h = compressHash(h, 20); logger->stdout(h.to_string(base, base == SRI)); } From 24544d3177324bd140d65db4068fda0b2701c6c9 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 2 Jun 2020 09:27:27 -0500 Subject: [PATCH 047/102] Verify added path matches calculated store path in add-to-store --- src/nix/add-to-store.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index ced8704ef0a..c96a16cebc8 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -61,8 +61,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(ingestionMethod, hash); - if (!dryRun) - store->addToStore(*namePart, path, ingestionMethod, git ? htSHA1 : htSHA256); + if (!dryRun) { + auto addedPath = store->addToStore(*namePart, path, ingestionMethod, git ? htSHA1 : htSHA256); + if (addedPath != info.path) + throw Error(format("Added path %s does not match calculated path %s; something has changed") % addedPath.to_string() % info.path.to_string()); + } logger->stdout("%s", store->printStorePath(info.path)); } From 88b5a98f7ac61dda9df10d006dfc07946ecb5a69 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 2 Jun 2020 18:54:52 -0500 Subject: [PATCH 048/102] Handle treeHash in Git fetcher Handles the treeHash attr in the Git fetcher. This can be used like: builtins.fetchTree { type = "git"; url = "https://github.com/nixos/nix.git"; treeHash = "0674cab54e1226d8e1e1d04572f07c5296edc862"; } Nix stores tree hash as the content hash in the store so that fetched objects are deduped. This uses the Git file ingestion method, which converts the files into git objects for hashing. --- src/libfetchers/git.cc | 101 +++++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 28 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 17cc602285c..93fdd0ba8a8 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" +#include "git.hh" #include @@ -27,6 +28,7 @@ struct GitInput : Input ParsedURL url; std::optional ref; std::optional rev; + std::optional treeHash; bool shallow = false; bool submodules = false; @@ -42,12 +44,13 @@ struct GitInput : Input other2 && url == other2->url && rev == other2->rev + && treeHash == other2->treeHash && ref == other2->ref; } bool isImmutable() const override { - return (bool) rev || narHash; + return (bool) rev || treeHash || narHash; } std::optional getRef() const override { return ref; } @@ -59,6 +62,7 @@ struct GitInput : Input ParsedURL url2(url); if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (treeHash) url2.query.insert_or_assign("treeHash", treeHash->gitRev()); if (ref) url2.query.insert_or_assign("ref", *ref); if (shallow) url2.query.insert_or_assign("shallow", "1"); return url2; @@ -72,6 +76,8 @@ struct GitInput : Input attrs.emplace("ref", *ref); if (rev) attrs.emplace("rev", rev->gitRev()); + if (treeHash) + attrs.emplace("treeHash", treeHash->gitRev()); if (shallow) attrs.emplace("shallow", true); if (submodules) @@ -96,6 +102,9 @@ struct GitInput : Input auto input = std::make_shared(*this); assert(!rev || rev->type == htSHA1); + assert(!treeHash || treeHash->type == htSHA1); + + auto ingestionMethod = treeHash ? FileIngestionMethod::Git : FileIngestionMethod::Recursive; std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -103,18 +112,23 @@ struct GitInput : Input auto getImmutableAttrs = [&]() { - return Attrs({ + Attrs attrs({ {"type", cacheType}, {"name", name}, - {"rev", input->rev->gitRev()}, }); + if (input->treeHash) + attrs.insert_or_assign("treeHash", input->treeHash->gitRev()); + else + attrs.insert_or_assign("rev", input->rev->gitRev()); + return attrs; }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) -> std::pair> { - assert(input->rev); + assert(input->rev || input->treeHash); assert(!rev || rev == input->rev); + assert(!treeHash || treeHash == input->treeHash); return { Tree { .actualPath = store->toRealPath(storePath), @@ -138,7 +152,7 @@ struct GitInput : Input // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. - if (!input->ref && !input->rev && isLocal) { + if (!input->ref && !input->rev && !input->treeHash && isLocal) { bool clean = false; /* Check whether this repo has any commits. There are @@ -195,7 +209,7 @@ struct GitInput : Input return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore("source", actualUrl, ingestionMethod, htSHA256, filter); auto tree = Tree { .actualPath = store->printStorePath(storePath), @@ -224,7 +238,7 @@ struct GitInput : Input if (isLocal) { - if (!input->rev) + if (!input->rev && !input->treeHash) input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); repoDir = actualUrl; @@ -233,12 +247,20 @@ struct GitInput : Input if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { + if (!input->rev || rev == rev2) { input->rev = rev2; return makeResult(res->first, std::move(res->second)); } } + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto treeHash2 = Hash(getStrAttr(res->first, "treeHash"), htSHA1); + if (!input->treeHash || treeHash == treeHash2) { + input->treeHash = treeHash2; + return makeResult(res->first, std::move(res->second)); + } + } + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); repoDir = cacheDir; @@ -255,11 +277,12 @@ struct GitInput : Input bool doFetch; time_t now = time(0); - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (input->rev) { + /* If a rev or treeHash is specified, we need to fetch if + it's not in the repo. */ + if (input->rev || input->treeHash) { try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + auto gitHash = input->treeHash ? input->treeHash : input->rev; + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", gitHash->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -297,7 +320,7 @@ struct GitInput : Input utimes(localRefFile.c_str(), times); } - if (!input->rev) + if (!input->rev && !input->treeHash) input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); } @@ -308,7 +331,10 @@ struct GitInput : Input // FIXME: check whether rev is an ancestor of ref. - printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + if (input->rev) + printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + else if (input->treeHash) + printTalkative("using tree %s of repo '%s'", input->treeHash->gitRev(), actualUrl); /* Now that we know the ref, check again whether we have it in the store. */ @@ -319,6 +345,9 @@ struct GitInput : Input AutoDelete delTmpDir(tmpDir, true); PathFilter filter = defaultPathFilter; + if (submodules && treeHash) + throw Error("Cannot combine tree hashes with git submodules"); + if (submodules) { Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); @@ -330,7 +359,7 @@ struct GitInput : Input runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->treeHash ? input->treeHash->gitRev() : input->rev->gitRev() }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); @@ -339,7 +368,7 @@ struct GitInput : Input // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + RunOptions gitOptions("git", { "-C", repoDir, "archive", input->treeHash ? input->treeHash->gitRev() : input->rev->gitRev() }); gitOptions.standardOut = &sink; runProgram2(gitOptions); }); @@ -347,20 +376,33 @@ struct GitInput : Input unpackTarfile(*source, tmpDir); } - auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(name, tmpDir, ingestionMethod, ingestionMethod == FileIngestionMethod::Git ? htSHA1 : htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + // verify treeHash is what we actually obtained in the nix store + if (input->treeHash) { + auto path = store->toRealPath(store->printStorePath(storePath)); + auto gotHash = dumpGitHash(htSHA1, path); + if (gotHash != input->treeHash) + throw Error("Git hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), path, input->treeHash->gitRev(), gotHash.gitRev()); + } - Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, - {"lastModified", lastModified}, - }); + Attrs infoAttrs({}); + if (input->treeHash) { + infoAttrs.insert_or_assign("treeHash", input->treeHash->gitRev()); + infoAttrs.insert_or_assign("revCount", 0); + infoAttrs.insert_or_assign("lastModified", 0); + } else { + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + infoAttrs.insert_or_assign("lastModified", lastModified); + infoAttrs.insert_or_assign("rev", input->rev->gitRev()); - if (!shallow) - infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + if (!shallow) + infoAttrs.insert_or_assign("revCount", + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + } - if (!this->rev) + if (!this->rev && !this->treeHash) getCache()->add( store, mutableAttrs, @@ -397,7 +439,7 @@ struct GitInputScheme : InputScheme attrs.emplace("type", "git"); for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") + if (name == "rev" || name == "ref" || name == "treeHash") attrs.emplace(name, value); else url2.query.emplace(name, value); @@ -413,7 +455,7 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "treeHash") throw Error("unsupported Git input attribute '%s'", name); auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); @@ -425,6 +467,9 @@ struct GitInputScheme : InputScheme if (auto rev = maybeGetStrAttr(attrs, "rev")) input->rev = Hash(*rev, htSHA1); + if (auto treeHash = maybeGetStrAttr(attrs, "treeHash")) + input->treeHash = Hash(*treeHash, htSHA1); + input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false); From 7f77ba17d498a04a57f39d8fb984e1388f1ca6a2 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 3 Jun 2020 16:08:58 -0500 Subject: [PATCH 049/102] Support null bytes in hashes for dumpGitTree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit we need to keep this in vector form so that we don’t accidentally skip strings starting with 0x00. --- src/libutil/git.cc | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 3e613949a26..dfc9b5de8e7 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -154,7 +154,8 @@ GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink) GitMode dumpGitTree(const GitTree & entries, Sink & sink) { - std::string s1 = ""; + vector v1; + for (auto & i : entries) { unsigned int mode; switch (i.second.first) { @@ -162,14 +163,20 @@ GitMode dumpGitTree(const GitTree & entries, Sink & sink) case GitMode::Executable: mode = 100755; break; case GitMode::Regular: mode = 100644; break; } - s1 += (format("%06d %s\0%s"s) % mode % i.first % i.second.second.hash).str(); + auto s1 = (format("%06d %s") % mode % i.first).str(); + std::copy(s1.begin(), s1.end(), std::back_inserter(v1)); + v1.push_back(0); + std::copy(i.second.second.hash, i.second.second.hash + 20, std::back_inserter(v1)); } - std::string s2 = (format("tree %d\0%s"s) % s1.size() % s1).str(); + vector v2; + auto s2 = (format("tree %d"s) % v1.size()).str(); + std::copy(s2.begin(), s2.end(), std::back_inserter(v2)); + v2.push_back(0); + std::copy(v1.begin(), v1.end(), std::back_inserter(v2)); + + sink(v2.data(), v2.size()); - vector v; - std::copy(s2.begin(), s2.end(), std::back_inserter(v)); - sink(v.data(), v.size()); return GitMode::Directory; } From 6611e362caae2f0dc500457e526422b3f41c319c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 3 Jun 2020 16:30:53 -0500 Subject: [PATCH 050/102] =?UTF-8?q?Use=20=E2=80=9Cgit=E2=80=9D=20for=20all?= =?UTF-8?q?=20git=20object=20names?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git trees can have objects starting with “.” and other cases that Nix store names can’t have. To avoid having to create some escaping format, just use git for the name everywhere. The tree object has the real name. --- src/libstore/local-store.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libutil/git.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 14c816e7282..d1079ca4c30 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1143,7 +1143,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, throw SysError(format("getting attributes of path '%1%'") % srcPath); if (S_ISDIR(st.st_mode)) for (auto & i : readDirectory(srcPath)) - addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair); + addToStore("git", srcPath + "/" + i.name, method, hashAlgo, filter, repair); dumpGit(hashAlgo, srcPath, sink, filter); break; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index fbc161cc92c..31ede2f2fcf 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -501,7 +501,7 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, throw SysError(format("getting attributes of path '%1%'") % srcPath); if (S_ISDIR(st.st_mode)) for (auto & i : readDirectory(srcPath)) - addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair); + addToStore("git", srcPath + "/" + i.name, method, hashAlgo, filter, repair); } auto conn(getConnection()); diff --git a/src/libutil/git.cc b/src/libutil/git.cc index dfc9b5de8e7..72ed922f92c 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -112,7 +112,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa Hash hash(htSHA1); std::copy(hashs.begin(), hashs.end(), hash.hash); - string entryName = getStoreEntry(storeDir, hash, name); + string entryName = getStoreEntry(storeDir, hash, "git"); Path entry = absPath(realStoreDir + "/" + entryName); struct stat st; From d07212c0565eacf247150b91a11ed53646eb4ff6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 3 Jun 2020 17:11:41 -0500 Subject: [PATCH 051/102] Show error when git treeHash is not a tree --- src/libfetchers/git.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 93fdd0ba8a8..2d09197b03e 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -324,6 +324,12 @@ struct GitInput : Input input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); } + if (input->treeHash) { + auto type = chomp(runProgram("git", true, { "-C", repoDir, "cat-file", "-t", input->treeHash->gitRev() })); + if (type != "tree") + throw Error(format("Need a tree object, found '%s' object in %s") % type % input->treeHash->gitRev()); + } + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; if (isShallow && !shallow) From 3457b3eba50703a0d458e4644182a0a7ae74f345 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 3 Jun 2020 17:31:21 -0500 Subject: [PATCH 052/102] Copy executable bit correctly in fs-sink --- src/libutil/fs-sink.hh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index efd13685df3..e07695064d3 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -101,7 +101,10 @@ struct RestoreSink : ParseSink if (lstat(entry.c_str(), &st)) throw SysError(format("getting attributes of path '%1%'") % entry); if (S_ISREG(st.st_mode)) { - createRegularFile(destination + "/" + i.name); + if (st.st_mode & S_IXUSR) + createExecutableFile(destination + "/" + i.name); + else + createRegularFile(destination + "/" + i.name); copyFile(entry); } else if (S_ISDIR(st.st_mode)) copyDirectory(entry, destination + "/" + i.name); From 3e8ddacdb74b54e68733db6b6e90b353129ca647 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 3 Jun 2020 18:07:39 -0500 Subject: [PATCH 053/102] =?UTF-8?q?Don=E2=80=99t=20add=200=20in=20front=20?= =?UTF-8?q?of=20dumpGitTree=20format?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this should just be 4000, not 04000. --- src/libutil/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 72ed922f92c..14dd0bf8d9d 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -163,7 +163,7 @@ GitMode dumpGitTree(const GitTree & entries, Sink & sink) case GitMode::Executable: mode = 100755; break; case GitMode::Regular: mode = 100644; break; } - auto s1 = (format("%06d %s") % mode % i.first).str(); + auto s1 = (format("%d %s") % mode % i.first).str(); std::copy(s1.begin(), s1.end(), std::back_inserter(v1)); v1.push_back(0); std::copy(i.second.second.hash, i.second.second.hash + 20, std::back_inserter(v1)); From 192ea2b99215ed0d9a1da1f90e19b072d4f21cf7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 3 Jun 2020 18:25:26 -0500 Subject: [PATCH 054/102] Observe correct git sort order test/ should come *after* test.sh see http://git.661346.n2.nabble.com/In-tree-object-Must-the-td7446900.html#a7447657 --- src/libutil/git.cc | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 14dd0bf8d9d..3c49530f435 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -163,7 +163,10 @@ GitMode dumpGitTree(const GitTree & entries, Sink & sink) case GitMode::Executable: mode = 100755; break; case GitMode::Regular: mode = 100644; break; } - auto s1 = (format("%d %s") % mode % i.first).str(); + auto name = i.first; + if (i.second.first == GitMode::Directory) + name.pop_back(); + auto s1 = (format("%d %s") % mode % name).str(); std::copy(s1.begin(), s1.end(), std::back_inserter(v1)); v1.push_back(0); std::copy(i.second.second.hash, i.second.second.hash + 20, std::back_inserter(v1)); @@ -194,8 +197,17 @@ static GitMode dumpGitInternal(HashType ht, const Path & path, Sink & sink, Path else if (S_ISDIR(st.st_mode)) { GitTree entries; for (auto & i : readDirectory(path)) - if (filter(path + "/" + i.name)) - entries[i.name] = dumpGitHashInternal(ht, path + "/" + i.name, filter); + if (filter(path + "/" + i.name)) { + auto result = dumpGitHashInternal(ht, path + "/" + i.name, filter); + + // correctly observe git order, see + // https://github.com/mirage/irmin/issues/352 + auto name = i.name; + if (result.first == GitMode::Directory) + name += "/"; + + entries[name] = result; + } perm = dumpGitTree(entries, sink); } else throw Error(format("file '%1%' has an unsupported type") % path); From 35b0dff5fee27110b0093b9ca1ea459c870f2518 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Jun 2020 14:35:38 +0000 Subject: [PATCH 055/102] Somehow lost git tests --- tests/local.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/local.mk b/tests/local.mk index 536661af88e..af32cc8eb0c 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -4,6 +4,7 @@ nix_tests = \ gc.sh \ gc-concurrent.sh \ gc-auto.sh \ + git.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh \ From 7596005b9f37d38df6209dae657092b03fe5791f Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 4 Jun 2020 11:28:48 -0500 Subject: [PATCH 056/102] Add test for fetching git objects this uses the treeHash to fetch a created git object based on fetchGit.sh test --- tests/git.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/git.sh b/tests/git.sh index e700dcc785c..a09ee85d7e2 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -1,5 +1,7 @@ source common.sh +clearStore + try () { hash=$(nix hash-git --base16 --type sha1 $TEST_ROOT/hash-path) if test "$hash" != "$1"; then @@ -34,3 +36,31 @@ echo Hello World! > $TEST_ROOT/dummy3/hello/hello path3=$(nix add-to-store --git $TEST_ROOT/dummy3) hash3=$(nix-store -q --hash $path3) test "$hash3" = "sha256:1i2x80840igikhbyy7nqf08ymx3a6n83x1fzyrxvddf0sdl5nqvp" + +if [[ -n $(type -p git) ]]; then + repo=$TEST_ROOT/git + + export _NIX_FORCE_HTTP=1 + + rm -rf $repo $TEST_HOME/.cache/nix + + git init $repo + git -C $repo config user.email "foobar@example.com" + git -C $repo config user.name "Foobar" + + echo utrecht > $repo/hello + touch $repo/.gitignore + git -C $repo add hello .gitignore + git -C $repo commit -m 'Bla1' + + echo world > $repo/hello + git -C $repo commit -m 'Bla2' -a + + treeHash=$(git -C $repo rev-parse HEAD:) + + # Fetch the default branch. + path=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file://$repo; treeHash = \"$treeHash\"; }).outPath") + [[ $(cat $path/hello) = world ]] +else + echo "Git not installed; skipping Git tests" +fi From 21d9928c6c5a16fa63834323b5f8954369f60b65 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 4 Jun 2020 14:45:59 -0500 Subject: [PATCH 057/102] Add test for submodules in git --- tests/git.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/git.sh b/tests/git.sh index a09ee85d7e2..bdc98844a0d 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -61,6 +61,9 @@ if [[ -n $(type -p git) ]]; then # Fetch the default branch. path=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file://$repo; treeHash = \"$treeHash\"; }).outPath") [[ $(cat $path/hello) = world ]] + + # Submodules cause error. + (! nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file://$repo; treeHash = \"$treeHash\"; submodules = true; }).outPath") else echo "Git not installed; skipping Git tests" fi From 4a73bd046ba6343defdd31022ce4ca47fa5b4b87 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Jun 2020 23:09:38 +0000 Subject: [PATCH 058/102] `addToStore` should always use NAR hash `addToStoreFromDump` would use the CA-specific method. --- src/libstore/local-store.cc | 5 +---- src/nix/add-to-store.cc | 29 ++++++++++++++++++++--------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 69d3997778e..fe083b8e878 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1005,10 +1005,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, return n; }); - if (hasPrefix(info.ca, "fixed:git:")) - restoreGit(realPath, wrapperSource, realStoreDir, storeDir); - else - restorePath(realPath, wrapperSource); + restorePath(realPath, wrapperSource); auto hashResult = hashSink->finish(); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index b70eeccd8db..9541e5d1ac9 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -10,7 +10,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - bool git = false; + FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -26,8 +26,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand addFlag({ .longName = "git", + .shortName = 0, .description = "treat path as a git object", - .handler = {&this->git, true}, + .handler = {&ingestionMethod, FileIngestionMethod::Git}, }); } @@ -48,13 +49,25 @@ struct CmdAddToStore : MixDryRun, StoreCommand { if (!namePart) namePart = baseNameOf(path); - auto ingestionMethod = git ? FileIngestionMethod::Git : FileIngestionMethod::Recursive; - StringSink sink; dumpPath(path, sink); auto narHash = hashString(htSHA256, *sink.s); - auto hash = git ? dumpGitHash(htSHA1, path) : narHash; + + Hash hash; + switch (ingestionMethod) { + case FileIngestionMethod::Recursive: { + hash = narHash; + break; + } + case FileIngestionMethod::Flat: { + abort(); // not yet supported above + } + case FileIngestionMethod::Git: { + hash = dumpGitHash(htSHA1, path); + break; + } + } ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); info.narHash = narHash; @@ -62,10 +75,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand info.ca = makeFixedOutputCA(ingestionMethod, hash); if (!dryRun) { - auto addedPath = store->addToStore(*namePart, path, ingestionMethod, git ? htSHA1 : htSHA256); - if (addedPath != info.path) - throw Error("Added path %s does not match calculated path %s; something has changed", - addedPath.to_string(), info.path.to_string()); + auto source = StringSource { *sink.s }; + store->addToStore(info, source); } logger->stdout("%s", store->printStorePath(info.path)); From 74b83c4e8e56e78d027275f4bd021b1f985468ae Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 18 Jun 2020 22:26:52 -0400 Subject: [PATCH 059/102] Try to substitute builtins.fetch* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit builtins.fetch* aren’t technically derivations, so they never got substituted. To work around this, we can try to substitute this store path during evaluation. This can happen any time a hash is provided. Updated to be closer to how the flakes branch works. --- src/libexpr/primops/fetchTree.cc | 14 ++++++++++++++ src/libfetchers/fetchers.cc | 18 ++++++++++++++++++ src/libfetchers/fetchers.hh | 3 +++ 3 files changed, 35 insertions(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9be93710ad3..6725ab22e4f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -134,6 +134,20 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + // try to substitute if we can + if (settings.useSubstitutes && expectedHash) { + auto substitutableStorePath = fetchers::trySubstitute(state.store, + unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, *expectedHash, name); + if (substitutableStorePath) { + auto substitutablePath = state.store->toRealPath(*substitutableStorePath); + if (state.allowedPaths) + state.allowedPaths->insert(substitutablePath); + + mkString(v, substitutablePath, PathSet({substitutablePath})); + return; + } + } + auto storePath = unpack ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 9174c3de467..ec0510652f4 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -72,4 +72,22 @@ std::pair> Input::fetchTree(ref store) return {std::move(tree), input}; } +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name) +{ + auto substitutablePath = store->makeFixedOutputPath(ingestionMethod, hash, name); + + try { + store->ensurePath(substitutablePath); + + debug("using substituted path '%s'", store->printStorePath(substitutablePath)); + + return substitutablePath; + } catch (Error & e) { + debug("substitution of path '%s' failed: %s", store->printStorePath(substitutablePath), e.what()); + } + + return std::nullopt; +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 59a58ae6781..2eeaed2f287 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -100,4 +100,7 @@ Tree downloadTarball( const std::string & name, bool immutable); +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name); + } From 5fa33d70a85f0a29e218d38dd6ced6c4086332de Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 22 Jun 2020 18:08:24 -0400 Subject: [PATCH 060/102] Support substituting in fetchGit --- src/libfetchers/git.cc | 20 +++++++++++++++++++- tests/git.sh | 9 +++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 85902126dbf..49587cf461b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -106,6 +106,24 @@ struct GitInput : Input auto ingestionMethod = treeHash ? FileIngestionMethod::Git : FileIngestionMethod::Recursive; + // try to substitute + if (settings.useSubstitutes && treeHash && !submodules) { + auto storePath = fetchers::trySubstitute(store, ingestionMethod, *treeHash, name); + if (storePath) { + return { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + .info = TreeInfo { + .revCount = std::nullopt, + .lastModified = 0, + }, + }, + input + }; + } + } + std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; if (submodules) cacheType += "-submodules"; @@ -385,7 +403,7 @@ struct GitInput : Input unpackTarfile(*source, tmpDir); } - auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, ingestionMethod == FileIngestionMethod::Git ? htSHA1 : htSHA256, filter); + auto storePath = store->addToStore(name, tmpDir, ingestionMethod, ingestionMethod == FileIngestionMethod::Git ? htSHA1 : htSHA256, filter); // verify treeHash is what we actually obtained in the nix store if (input->treeHash) { diff --git a/tests/git.sh b/tests/git.sh index bdc98844a0d..cd9979ada29 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -1,6 +1,7 @@ source common.sh clearStore +clearCache try () { hash=$(nix hash-git --base16 --type sha1 $TEST_ROOT/hash-path) @@ -40,8 +41,6 @@ test "$hash3" = "sha256:1i2x80840igikhbyy7nqf08ymx3a6n83x1fzyrxvddf0sdl5nqvp" if [[ -n $(type -p git) ]]; then repo=$TEST_ROOT/git - export _NIX_FORCE_HTTP=1 - rm -rf $repo $TEST_HOME/.cache/nix git init $repo @@ -64,6 +63,12 @@ if [[ -n $(type -p git) ]]; then # Submodules cause error. (! nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file://$repo; treeHash = \"$treeHash\"; submodules = true; }).outPath") + + # Check that we can substitute it from other places. + nix copy --to file://$cacheDir $path + nix-store --delete $path + path2=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$treeHash\"; }).outPath" --substituters file://$cacheDir) + [ $path2 = $path ] else echo "Git not installed; skipping Git tests" fi From 4074142e688ee156739461343a460b838ea48e92 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 13:34:04 -0400 Subject: [PATCH 061/102] Override substitute option in test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is disabled automatically when we’re offline. --- tests/git.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/git.sh b/tests/git.sh index cd9979ada29..85ce99f6c40 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -67,7 +67,7 @@ if [[ -n $(type -p git) ]]; then # Check that we can substitute it from other places. nix copy --to file://$cacheDir $path nix-store --delete $path - path2=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$treeHash\"; }).outPath" --substituters file://$cacheDir) + path2=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$treeHash\"; }).outPath" --substituters file://$cacheDir --option substitute true) [ $path2 = $path ] else echo "Git not installed; skipping Git tests" From e1f836bdc7ef0b83c8f9a54698df5a7cf42d0c0e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jun 2020 00:06:00 -0400 Subject: [PATCH 062/102] Store CA for fetchers --- src/libfetchers/fetchers.cc | 9 ++++++++- src/libfetchers/git.cc | 10 ++++++++++ src/libfetchers/tree-info.hh | 3 +++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index ec0510652f4..925874a0669 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "store-api.hh" +#include "archive.hh" #include @@ -60,7 +61,13 @@ std::pair> Input::fetchTree(ref store) tree.actualPath = store->toRealPath(tree.storePath); if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + tree.info.narHash = store->queryPathInfo(tree.storePath, tree.info.ca)->narHash; + + if (!tree.info.narHash) { + HashSink hashSink(htSHA256); + dumpPath(tree.actualPath, hashSink); + tree.info.narHash = hashSink.finish().first; + } if (input->narHash) assert(input->narHash == tree.info.narHash); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 49587cf461b..3ef8f6ec2c7 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -106,6 +106,13 @@ struct GitInput : Input auto ingestionMethod = treeHash ? FileIngestionMethod::Git : FileIngestionMethod::Recursive; + std::optional ca; + if (treeHash) + ca = FixedOutputHash { + .method = ingestionMethod, + .hash = *treeHash, + }; + // try to substitute if (settings.useSubstitutes && treeHash && !submodules) { auto storePath = fetchers::trySubstitute(store, ingestionMethod, *treeHash, name); @@ -115,6 +122,7 @@ struct GitInput : Input .actualPath = store->toRealPath(*storePath), .storePath = std::move(*storePath), .info = TreeInfo { + .ca = ca, .revCount = std::nullopt, .lastModified = 0, }, @@ -152,6 +160,7 @@ struct GitInput : Input .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), .info = TreeInfo { + .ca = ca, .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), .lastModified = getIntAttr(infoAttrs, "lastModified"), }, @@ -233,6 +242,7 @@ struct GitInput : Input .actualPath = store->printStorePath(storePath), .storePath = std::move(storePath), .info = TreeInfo { + .ca = ca, // FIXME: maybe we should use the timestamp of the last // modified dirty file? .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh index 2c734728135..a75bb9c77cc 100644 --- a/src/libfetchers/tree-info.hh +++ b/src/libfetchers/tree-info.hh @@ -12,6 +12,9 @@ namespace nix::fetchers { struct TreeInfo { Hash narHash; + + std::optional ca; + std::optional revCount; std::optional lastModified; From 3306877f03b990d6fd6c4eb4fc460576831fe336 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jun 2020 04:40:53 +0000 Subject: [PATCH 063/102] Temp skip CA argument on this branch --- src/libfetchers/fetchers.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 925874a0669..e477723f587 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -61,7 +61,7 @@ std::pair> Input::fetchTree(ref store) tree.actualPath = store->toRealPath(tree.storePath); if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath, tree.info.ca)->narHash; + tree.info.narHash = store->queryPathInfo(tree.storePath /*, tree.info.ca */)->narHash; if (!tree.info.narHash) { HashSink hashSink(htSHA256); From a24ebd21e89f8277a7a7467a70f5523bac0ae77a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jun 2020 22:18:40 +0000 Subject: [PATCH 064/102] Use `FixedOutputHash` in parameters for `makeFixedOutputPath` --- src/libexpr/primops.cc | 10 ++++++++-- src/libfetchers/tarball.cc | 9 +++++---- src/libfetchers/tree-info.cc | 5 ++++- src/libstore/binary-cache-store.cc | 5 ++++- src/libstore/build.cc | 5 ++++- src/libstore/content-address.cc | 15 +++++---------- src/libstore/content-address.hh | 4 ---- src/libstore/local-store.cc | 10 +++++----- src/libstore/store-api.cc | 20 +++++++++++--------- src/libstore/store-api.hh | 13 +++++++++---- src/nix-prefetch-url/nix-prefetch-url.cc | 16 +++++++++++----- src/nix-store/nix-store.cc | 5 ++++- src/nix/add-to-store.cc | 5 ++++- src/nix/make-content-addressable.cc | 5 ++++- 14 files changed, 78 insertions(+), 49 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3830d8107f1..ea85ad2ba27 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -772,7 +772,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputHash { + .method = ingestionMethod, + .hash = h + }); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { .path = std::move(outPath), @@ -1153,7 +1156,10 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = expectedHash, + }); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index f5356f0af8b..7f9b4d434ea 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -67,13 +67,14 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = FixedOutputHash { + FixedOutputHash foh { .method = FileIngestionMethod::Flat, .hash = hash, }; + ValidPathInfo info(store->makeFixedOutputPath(name, foh)); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = std::move(foh); auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index b2d8cfc8d03..39642352969 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -8,7 +8,10 @@ namespace nix::fetchers { StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); + return store.makeFixedOutputPath("source", FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }); } } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 7167ec90076..56f7e100324 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -347,7 +347,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(method, h, name)); + ValidPathInfo info(makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = h, + })); auto source = StringSource { *sink.s }; addToStore(info, source, repair, CheckSigs, nullptr); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1474f51745b..821921be90d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3733,7 +3733,10 @@ void DerivationGoal::registerOutputs() ? hashPath(*i.second.hash->hash.type, actualPath).first : hashFile(*i.second.hash->hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(i.second.path.name(), FixedOutputHash { + .method = i.second.hash->method, + .hash = h2, + }); if (i.second.hash->hash != h2) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3d753836f8a..ebd1763f522 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -17,13 +17,6 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { } } -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) -{ - return "fixed:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base32, true); -} - // FIXME Put this somewhere? template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; @@ -31,10 +24,13 @@ template overloaded(Ts...) -> overloaded; std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { - return "text:" + th.hash.to_string(Base32, true); + return "text:" + + th.hash.to_string(Base32, true); }, [](FixedOutputHash fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } @@ -51,7 +47,6 @@ ContentAddress parseContentAddress(std::string_view rawCa) { } return TextHash { hash }; } else if (prefix == "fixed") { - // This has to be an inverse of makeFixedOutputCA auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos); if (methodAndHash.substr(0,2) == "r:") { std::string_view hashRaw = methodAndHash.substr(2,string::npos); diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index ba4797f5b00..1c58fa7355c 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -41,10 +41,6 @@ typedef std::variant< ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional ca); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c266ee5fd61..05d1f88acfd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -562,10 +562,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); check( - makeFixedOutputPath( - out->second.hash->method, - out->second.hash->hash, - drvName), + makeFixedOutputPath(drvName, *out->second.hash), out->second.path, "out"); } @@ -1050,7 +1047,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam { Hash h = hashString(hashAlgo, dump); - auto dstPath = makeFixedOutputPath(method, h, name); + auto dstPath = makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = h, + }); addTempRoot(dstPath); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d8a03bc3b4f..92fc02aa96b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -173,21 +173,20 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, std::string_view name, + const FixedOutputHash & foh, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (*foh.hash.type == htSHA256 && foh.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", references, hasSelfReference), foh.hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(foh.method) + + foh.hash.to_string(Base16, true) + ":"), name); } } @@ -205,7 +204,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress return makeTextPath(name, th.hash, references); }, [&](FixedOutputHash fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + return makeFixedOutputPath(name, fsh, references, hasSelfReference); } }, ca); } @@ -227,7 +226,10 @@ std::pair Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + return std::make_pair(makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = h, + }), h); } @@ -835,7 +837,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const return store.makeTextPath(path.name(), th.hash, references); }, [&](FixedOutputHash fsh) { - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference); + return store.makeFixedOutputPath(path.name(), fsh, references, hasSelfReference); } }, *ca); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 33e607ed37f..b3a9cff367e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -250,15 +250,20 @@ public: StorePath makeOutputPath(const string & id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, + StorePath makeFixedOutputPath( + std::string_view name, + const FixedOutputHash & hash, const StorePathSet & references = {}, bool hasSelfReference = false) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, + StorePath makeTextPath( + std::string_view name, + const Hash & hash, const StorePathSet & references = {}) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, + StorePath makeFixedOutputPathFromCA( + std::string_view name, + ContentAddress ca, const StorePathSet & references = {}, bool hasSelfReference = false) const; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f39e..64f2f956a8e 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -157,8 +157,11 @@ static int _main(int argc, char * * argv) std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, expectedHash, name); + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = expectedHash, + }); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -207,15 +210,18 @@ static int _main(int argc, char * * argv) if (expectedHash != Hash(ht) && expectedHash != hash) throw Error("hash mismatch for '%1%'", uri); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; /* Copy the file to the Nix store. FIXME: if RemoteStore implemented addToStoreFromDump() and downloadFile() supported a sink, we could stream the download directly into the Nix store. */ - storePath = store->addToStore(name, tmpFile, recursive, ht); + storePath = store->addToStore(name, tmpFile, method, ht); - assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); + assert(*storePath == store->makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = expectedHash, + })); } stopProgressBar(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c4ca89c8558..43f83b4d2a5 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -208,7 +208,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputHash { + .method = recursive, + .hash = Hash { hash, hashAlgo }, + }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 48a4722aec3..fce7d3510ff 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -67,7 +67,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand } } - ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); + ValidPathInfo info(store->makeFixedOutputPath(*namePart, FixedOutputHash { + .method = ingestionMethod, + .hash = hash, + })); info.narHash = narHash; info.narSize = sink.s->size(); info.ca = std::optional { FixedOutputHash { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 5267948eeb9..47c6f1f3e77 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -77,7 +77,10 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); + ValidPathInfo info(store->makeFixedOutputPath(path.name(), FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, references, hasSelfReference)); info.references = std::move(references); info.hasSelfReference = std::move(hasSelfReference); info.narHash = narHash; From 69fe58c90155cdbf1082aa0c96d1f47c750b925e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jun 2020 20:14:36 +0000 Subject: [PATCH 065/102] Use `PathReferences` in a few more places --- src/libstore/path-info.hh | 16 +++++++---- src/libstore/store-api.cc | 43 ++++++++++++++++++----------- src/libstore/store-api.hh | 6 ++-- src/nix/make-content-addressable.cc | 39 ++++++++++++-------------- 4 files changed, 56 insertions(+), 48 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 27efe5ae93f..f1d57c85325 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -26,9 +26,9 @@ struct PathReferences std::set references; bool hasSelfReference = false; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); }; @@ -116,9 +116,9 @@ struct ValidPathInfo : PathReferences /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; void insertReferencePossiblyToSelf(StorePath && ref); void setReferencesPossiblyToSelf(StorePathSet && refs); @@ -139,6 +139,10 @@ struct ValidPathInfo : PathReferences ValidPathInfo(StorePath && path) : path(std::move(path)) { }; ValidPathInfo(const StorePath & path) : path(path) { }; + ValidPathInfo(const Store & store, + std::string_view name, + ContentAddress && ca, PathReferences && refs); + virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 92fc02aa96b..9a645ec7da7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -160,14 +160,13 @@ StorePath Store::makeOutputPath(const string & id, static std::string makeType( const Store & store, string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const PathReferences & references) { - for (auto & i : references) { + for (auto & i : references.references) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.hasSelfReference) type += ":self"; return std::move(type); } @@ -175,13 +174,13 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( std::string_view name, const FixedOutputHash & foh, - const StorePathSet & references, - bool hasSelfReference) const + const PathReferences & references) const { if (*foh.hash.type == htSHA256 && foh.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), foh.hash, name); + return makeStorePath(makeType(*this, "source", references), foh.hash, name); } else { - assert(references.empty()); + assert(references.references.empty()); + assert(!references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" @@ -196,15 +195,16 @@ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + const PathReferences & references) const { // New template return std::visit(overloaded { [&](TextHash th) { - return makeTextPath(name, th.hash, references); + assert(!references.hasSelfReference); + return makeTextPath(name, th.hash, references.references); }, [&](FixedOutputHash fsh) { - return makeFixedOutputPath(name, fsh, references, hasSelfReference); + return makeFixedOutputPath(name, fsh, references); } }, ca); } @@ -216,7 +216,7 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, /* Stuff the references (if any) into the type. This is a bit hacky, but we can't put them in `s' since that would be ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + return makeStorePath(makeType(*this, "text", PathReferences { references }), hash, name); } @@ -796,17 +796,17 @@ string showPaths(const PathSet & paths) StorePathSet ValidPathInfo::referencesPossiblyToSelf() const { - return PathReferences::referencesPossiblyToSelf(path); + return PathReferences::referencesPossiblyToSelf(path); } void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) { - return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); + return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); } void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) { - return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); + return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); } std::string ValidPathInfo::fingerprint(const Store & store) const @@ -837,7 +837,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const return store.makeTextPath(path.name(), th.hash, references); }, [&](FixedOutputHash fsh) { - return store.makeFixedOutputPath(path.name(), fsh, references, hasSelfReference); + return store.makeFixedOutputPath(path.name(), fsh, *this); } }, *ca); @@ -877,6 +877,17 @@ Strings ValidPathInfo::shortRefs() const } +ValidPathInfo::ValidPathInfo( + const Store & store, + std::string_view name, + ContentAddress && ca, + PathReferences && refs) + : PathReferences(std::move(refs)) + , path(store.makeFixedOutputPathFromCA(name, ca, *this)) + , ca(std::move(ca)) +{ +} + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b3a9cff367e..eaff7477c8c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -253,8 +253,7 @@ public: StorePath makeFixedOutputPath( std::string_view name, const FixedOutputHash & hash, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + const PathReferences & references = {}) const; StorePath makeTextPath( std::string_view name, @@ -264,8 +263,7 @@ public: StorePath makeFixedOutputPathFromCA( std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + const PathReferences & references = {}) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 47c6f1f3e77..f5fbca8e724 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -55,19 +55,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + PathReferences refs; + refs.hasSelfReference = oldInfo->hasSelfReference; for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); + refs.references.insert(std::move(replacement)); } *sink.s = rewriteStrings(*sink.s, rewrites); @@ -77,18 +73,17 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; - ValidPathInfo info(store->makeFixedOutputPath(path.name(), FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, references, hasSelfReference)); - info.references = std::move(references); - info.hasSelfReference = std::move(hasSelfReference); + ValidPathInfo info { + *store, + path.name(), + FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + std::move(refs), + }; info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); From 783e262cecf4cb845e1f238b034805c8e4702ed3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jun 2020 22:50:57 +0000 Subject: [PATCH 066/102] CA: Distingish Full vs Mini content addresses --- src/libexpr/primops.cc | 14 +-- src/libexpr/value-to-xml.hh | 2 +- src/libfetchers/tarball.cc | 15 ++- src/libfetchers/tree-info.cc | 7 +- src/libstore/binary-cache-store.cc | 7 +- src/libstore/build.cc | 40 ++++--- src/libstore/content-address.cc | 16 +-- src/libstore/content-address.hh | 85 ++++++++++++++- src/libstore/daemon.cc | 19 ++-- src/libstore/legacy-ssh-store.cc | 4 +- src/libstore/local-store.cc | 99 ++++++++++-------- src/libstore/local-store.hh | 3 +- src/libstore/misc.cc | 18 +++- src/libstore/nar-info-disk-cache.cc | 4 +- src/libstore/nar-info.cc | 4 +- src/libstore/path-info.hh | 48 +-------- src/libstore/path.hh | 3 - src/libstore/remote-store.cc | 59 ++++++----- src/libstore/remote-store.hh | 3 +- src/libstore/store-api.cc | 126 +++++++++++++---------- src/libstore/store-api.hh | 20 ++-- src/libstore/worker-protocol.hh | 4 +- src/libutil/fmt.hh | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 14 +-- src/nix-store/nix-store.cc | 11 +- src/nix/add-to-store.cc | 19 ++-- src/nix/make-content-addressable.cc | 18 ++-- src/nix/path-info.cc | 2 +- 28 files changed, 380 insertions(+), 286 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ea85ad2ba27..3e220386b91 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -772,9 +772,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputHash { - .method = ingestionMethod, - .hash = h + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + ingestionMethod, + h, + {}, }); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { @@ -1156,9 +1157,10 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = expectedHash, + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + method, + expectedHash, + {}, }); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327edb..02ccd714a90 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,5 +10,5 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); - + } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 7f9b4d434ea..197ad642853 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -67,14 +67,19 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); - FixedOutputHash foh { - .method = FileIngestionMethod::Flat, - .hash = hash, + ValidPathInfo info { + *store, + FullContentAddress { + .name = name, + .info = FixedOutputInfo { + FileIngestionMethod::Flat, + hash, + {}, + }, + }, }; - ValidPathInfo info(store->makeFixedOutputPath(name, foh)); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); - info.ca = std::move(foh); auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index 39642352969..f77b95dea0e 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -8,9 +8,10 @@ namespace nix::fetchers { StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); - return store.makeFixedOutputPath("source", FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narHash, + return store.makeFixedOutputPath("source", FixedOutputInfo { + FileIngestionMethod::Recursive, + narHash, + {}, }); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 56f7e100324..8c9bfccb7bf 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -347,9 +347,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = h, + ValidPathInfo info(makeFixedOutputPath(name, FixedOutputInfo { + method, + h, + {}, })); auto source = StringSource { *sink.s }; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 821921be90d..f58724f3a10 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ class Worker GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -3714,7 +3714,7 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::optional ca; + std::optional ca; if (fixedOutput) { @@ -3733,9 +3733,10 @@ void DerivationGoal::registerOutputs() ? hashPath(*i.second.hash->hash.type, actualPath).first : hashFile(*i.second.hash->hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(i.second.path.name(), FixedOutputHash { - .method = i.second.hash->method, - .hash = h2, + auto dest = worker.store.makeFixedOutputPath(i.second.path.name(), FixedOutputInfo { + i.second.hash->method, + h2, + {}, // TODO references }); if (i.second.hash->hash != h2) { @@ -3767,9 +3768,13 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = FixedOutputHash { - .method = i.second.hash->method, - .hash = h2, + ca = FullContentAddress { + .name = std::string { i.second.path.name() }, + .info = FixedOutputInfo { + i.second.hash->method, + h2, + {}, + }, }; } @@ -3834,13 +3839,14 @@ void DerivationGoal::registerOutputs() worker.markContentsGood(worker.store.parseStorePath(path)); } - ValidPathInfo info(worker.store.parseStorePath(path)); + auto info = ca + ? ValidPathInfo { worker.store, FullContentAddress { *ca } } + : ValidPathInfo { worker.store.parseStorePath(path) }; info.narHash = hash.first; info.narSize = hash.second; info.setReferencesPossiblyToSelf(std::move(references)); info.deriver = drvPath; info.ultimate = true; - info.ca = ca; worker.store.signPathInfo(info); if (!info.references.empty()) { @@ -4268,6 +4274,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ + // TODO std::variant storePath; StorePath storePath; /* The remaining substituters. */ @@ -4304,10 +4311,11 @@ class SubstitutionGoal : public Goal GoalState state; /* Content address for recomputing store path */ - std::optional ca; + // TODO delete once `storePath` is variant. + std::optional ca; public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4337,7 +4345,7 @@ class SubstitutionGoal : public Goal }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) @@ -4418,7 +4426,7 @@ void SubstitutionGoal::tryNext() auto subPath = storePath; if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA(*ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } @@ -4545,7 +4553,7 @@ void SubstitutionGoal::tryToRun() auto subPath = storePath; if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA(*ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } @@ -4684,7 +4692,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index ebd1763f522..477b93c5bad 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -21,7 +21,7 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -std::string renderContentAddress(ContentAddress ca) { +std::string renderMiniContentAddress(MiniContentAddress ca) { return std::visit(overloaded { [](TextHash th) { return "text:" @@ -35,7 +35,7 @@ std::string renderContentAddress(ContentAddress ca) { }, ca); } -ContentAddress parseContentAddress(std::string_view rawCa) { +MiniContentAddress parseMiniContentAddress(std::string_view rawCa) { auto prefixSeparator = rawCa.find(':'); if (prefixSeparator != string::npos) { auto prefix = string(rawCa, 0, prefixSeparator); @@ -43,7 +43,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); if (*hash.type != htSHA256) { - throw Error("parseContentAddress: the text hash should have type SHA256"); + throw Error("parseMiniContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; } else if (prefix == "fixed") { @@ -62,19 +62,19 @@ ContentAddress parseContentAddress(std::string_view rawCa) { }; } } else { - throw Error("parseContentAddress: format not recognized; has to be text or fixed"); + throw Error("parseMiniContentAddress: format not recognized; has to be text or fixed"); } } else { throw Error("Not a content address because it lacks an appropriate prefix"); } }; -std::optional parseContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional {} : parseContentAddress(rawCaOpt); +std::optional parseMiniContentAddressOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional {} : parseMiniContentAddress(rawCaOpt); }; -std::string renderContentAddress(std::optional ca) { - return ca ? renderContentAddress(*ca) : ""; +std::string renderMiniContentAddress(std::optional ca) { + return ca ? renderMiniContentAddress(*ca) : ""; } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 1c58fa7355c..b10943da228 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -2,9 +2,14 @@ #include #include "hash.hh" +#include "path.hh" namespace nix { +/* + * Mini content address + */ + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true @@ -35,18 +40,88 @@ struct FixedOutputHash { typedef std::variant< TextHash, // for paths computed by makeTextPath() / addTextToStore FixedOutputHash // for path computed by makeFixedOutputPath -> ContentAddress; +> MiniContentAddress; /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -std::string renderContentAddress(ContentAddress ca); +std::string renderMiniContentAddress(MiniContentAddress ca); + +std::string renderMiniContentAddress(std::optional ca); + +MiniContentAddress parseMiniContentAddress(std::string_view rawCa); + +std::optional parseMiniContentAddressOpt(std::string_view rawCaOpt); + +/* + * References set + */ + +template +struct PathReferences +{ + std::set references; + bool hasSelfReference = false; + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); +}; + +template +StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet references { references }; + if (hasSelfReference) + references.insert(self); + return references; +} + +template +void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} -std::string renderContentAddress(std::optional ca); +template +void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); -ContentAddress parseContentAddress(std::string_view rawCa); + references = refs; +} + +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ -std::optional parseContentAddressOpt(std::string_view rawCaOpt); +// This matches the additional info that we need for makeTextPath +struct TextInfo : TextHash { + // References for the paths, self references disallowed + StorePathSet references; +}; + +struct FixedOutputInfo : FixedOutputHash { + // References for the paths + PathReferences references; +}; + +struct FullContentAddress { + std::string name; + std::variant< + TextInfo, + FixedOutputInfo + > info; +}; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d4d68e8801c..66ce8c2e26a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -593,7 +593,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); + store->querySubstitutablePathInfos({path}, {}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -610,15 +610,12 @@ static void performOp(TunnelLogger * logger, ref store, case wopQuerySubstitutablePathInfos: { SubstitutablePathInfos infos; - StorePathCAMap pathsMap = {}; - if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = readStorePaths(*store, from); - for (auto & path : paths) - pathsMap.emplace(path, std::nullopt); - } else - pathsMap = readStorePathCAMap(*store, from); + auto paths = readStorePaths(*store, from); + std::set caPaths; + if (GET_PROTOCOL_MINOR(clientVersion) > 22) { + caPaths = readFullCaSet(*store, from); logger->startWork(); - store->querySubstitutablePathInfos(pathsMap, infos); + store->querySubstitutablePathInfos(paths, caPaths, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { @@ -658,7 +655,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << renderContentAddress(info->ca); + << renderMiniContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -716,7 +713,7 @@ static void performOp(TunnelLogger * logger, ref store, info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseContentAddressOpt(readString(from)); + info.ca = parseMiniContentAddressOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index f01e642a035..e33aa46342b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - info->ca = parseContentAddressOpt(readString(conn->from)); + info->ca = parseMiniContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } @@ -145,7 +145,7 @@ struct LegacySSHStore : public Store << info.narSize << info.ultimate << info.sigs - << renderContentAddress(info.ca); + << renderMiniContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 05d1f88acfd..b56bab20187 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -562,7 +562,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); check( - makeFixedOutputPath(drvName, *out->second.hash), + makeFixedOutputPath(drvName, FixedOutputInfo { + *out->second.hash, + {}, + }), out->second.path, "out"); } @@ -589,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderContentAddress(info.ca), (bool) info.ca) + (renderMiniContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = sqlite3_last_insert_rowid(state.db); @@ -663,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = parseMiniContentAddressOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -688,7 +691,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderContentAddress(info.ca), (bool) info.ca) + (renderMiniContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } @@ -842,46 +845,53 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; + + auto query = [&](auto & sub, const StorePath & localPath, const StorePath & subPath) { + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + return; + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + infos.insert_or_assign(localPath, SubstitutablePathInfo { + info->references, + info->hasSelfReference, + info->deriver, + narInfo ? narInfo->fileSize : 0, + info->narSize, + }); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + }; + for (auto & sub : getDefaultSubstituters()) { for (auto & path : paths) { - auto subPath(path.first); - - // recompute store path so that we can use a different store root - if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo { - info->references, - info->hasSelfReference, - info->deriver, - narInfo ? narInfo->fileSize : 0, - info->narSize, - }); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } + if (sub->storeDir != storeDir) continue; + query(sub, path, path); + } + for (auto & ca : caPaths) { + // TODO Deal with references: either disallow, or require the + // store path lengths be the same and rewrite strings. + auto localPath = makeFixedOutputPathFromCA(ca); + auto subPath = sub->makeFixedOutputPathFromCA(ca); + if (sub->storeDir == storeDir) + assert(localPath == subPath); + if (localPath != subPath) + // TODO print CA too + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(localPath), sub->printStorePath(subPath), sub->getUri()); + query(sub, localPath, subPath); } } } @@ -1047,9 +1057,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam { Hash h = hashString(hashAlgo, dump); - auto dstPath = makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = h, + auto dstPath = makeFixedOutputPath(name, FixedOutputInfo { + method, + h, + {}, }); addTempRoot(dstPath); @@ -1127,7 +1138,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { hash, references }); addTempRoot(dstPath); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ab9fbfbe40c..660d7a08fe8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -139,7 +139,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index d1448aa342b..1097dfdd3cf 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -107,11 +107,15 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional getDerivationCA(const BasicDerivation & drv) +std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); - if (out != drv.outputs.end()) - return out->second.hash; + if (out != drv.outputs.end() && out->second.hash) { + return FullContentAddress { + .name = std::string { out->second.path.name() }, + .info = FixedOutputInfo { *out->second.hash, {} }, + }; + } return std::nullopt; } @@ -164,7 +168,11 @@ void Store::queryMissing(const std::vector & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); + auto caOpt = getDerivationCA(*drv); + if (caOpt) + querySubstitutablePathInfos({}, { *std::move(caOpt) }, infos); + else + querySubstitutablePathInfos({outPath}, {}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -221,7 +229,7 @@ void Store::queryMissing(const std::vector & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); + querySubstitutablePathInfos({path.path}, {}, infos); if (infos.empty()) { auto state(state_.lock()); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c543f6ea297..13924cc88c9 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = parseMiniContentAddressOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (renderContentAddress(info->ca)) + (renderMiniContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ef04bc85964..7005b26ec74 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -70,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseContentAddressOpt(value); + ca = parseMiniContentAddressOpt(value); } pos = eol + 1; @@ -107,7 +107,7 @@ std::string NarInfo::to_string(const Store & store) const res += "Sig: " + sig + "\n"; if (ca) - res += "CA: " + renderContentAddress(*ca) + "\n"; + res += "CA: " + renderMiniContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index f1d57c85325..f0d0e6b6172 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -20,47 +20,6 @@ namespace nix { class Store; -template -struct PathReferences -{ - std::set references; - bool hasSelfReference = false; - - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; - void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); - void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); -}; - -template -StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const -{ - StorePathSet references { references }; - if (hasSelfReference) - references.insert(self); - return references; -} - -template -void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) -{ - if (ref == self) - hasSelfReference = true; - else - references.insert(std::move(ref)); -} - -template -void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) -{ - if (refs.count(self)) - hasSelfReference = true; - refs.erase(self); - - references = refs; -} - struct ValidPathInfo : PathReferences { StorePath path; @@ -92,7 +51,7 @@ struct ValidPathInfo : PathReferences and the store path would be computed from the name component, ‘narHash’ and ‘references’. However, we support many types of content addresses. */ - std::optional ca; + std::optional ca; bool operator == (const ValidPathInfo & i) const { @@ -113,6 +72,8 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); + std::optional fullContentAddressOpt() const; + /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -140,8 +101,7 @@ struct ValidPathInfo : PathReferences ValidPathInfo(const StorePath & path) : path(path) { }; ValidPathInfo(const Store & store, - std::string_view name, - ContentAddress && ca, PathReferences && refs); + FullContentAddress && ca); virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index a07186323ad..c82ed9f266f 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,5 @@ #pragma once -#include "content-address.hh" #include "types.hh" namespace nix { @@ -63,8 +62,6 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; -typedef std::map> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4a9be444571..7e8f4922e0d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,22 +38,24 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -StorePathCAMap readStorePathCAMap(const Store & store, Source & from) +std::set readFullCaSet(const Store & store, Source & from) { - StorePathCAMap paths; - auto count = readNum(from); - while (count--) - paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); + std::set paths; + // TODO + // auto count = readNum(from); + // while (count--) + // paths.insert_or_assign(store.parseStorePath(readString(from)), parseMiniContentAddressOpt(readString(from))); return paths; } -void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths) +void writeFullCaSet(const Store & store, Sink & out, const std::set & paths) { - out << paths.size(); - for (auto & i : paths) { - out << store.printStorePath(i.first); - out << renderContentAddress(i.second); - } + // TODO + //out << paths.size(); + //for (auto & i : paths) { + // out << store.printStorePath(i.first); + // out << renderMiniContentAddress(i.second); + //} } @@ -326,43 +328,48 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { - if (pathsMap.empty()) return; + if (paths.empty() && caPaths.empty()) return; auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + auto combine = [&]() { + std::set combined { paths }; + for (auto & ca : caPaths) + combined.insert(makeFixedOutputPathFromCA(ca)); + return combined; + }; - for (auto & i : pathsMap) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + for (auto & path : combine()) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); + conn->to << wopQuerySubstitutablePathInfo << printStorePath(path); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(i.first, readStorePaths(*this, conn->from)); + info.setReferencesPossiblyToSelf(path, readStorePaths(*this, conn->from)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.first, std::move(info)); + infos.insert_or_assign(path, std::move(info)); } } else { conn->to << wopQuerySubstitutablePathInfos; if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { - StorePathSet paths; - for (auto & path : pathsMap) - paths.insert(path.first); + writeStorePaths(*this, conn->to, combine()); + } else { writeStorePaths(*this, conn->to, paths); - } else - writeStorePathCAMap(*this, conn->to, pathsMap); + writeFullCaSet(*this, conn->to, caPaths); + } conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - auto path = parseStorePath(readString(conn->from)); + auto path = parseStorePath(readString(conn->from)); SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") @@ -405,7 +412,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); + info->ca = parseMiniContentAddressOpt(readString(conn->from)); } } callback(std::move(info)); @@ -489,7 +496,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << renderContentAddress(info.ca) + << info.ultimate << info.sigs << renderMiniContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 07ed2b51a1d..7a3acd6bfa6 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -55,7 +55,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9a645ec7da7..91767e55f32 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -157,6 +157,9 @@ StorePath Store::makeOutputPath(const string & id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ static std::string makeType( const Store & store, string && type, @@ -171,52 +174,48 @@ static std::string makeType( } -StorePath Store::makeFixedOutputPath( - std::string_view name, - const FixedOutputHash & foh, - const PathReferences & references) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (*foh.hash.type == htSHA256 && foh.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references), foh.hash, name); + if (*info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(references.references.empty()); - assert(!references.hasSelfReference); + assert(info.references.references.size() == 0); + assert(!info.references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(foh.method) - + foh.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", PathReferences { info.references }), + info.hash, + name); +} + + // FIXME Put this somewhere? template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const PathReferences & references) const +StorePath Store::makeFixedOutputPathFromCA(const FullContentAddress & info) const { // New template return std::visit(overloaded { - [&](TextHash th) { - assert(!references.hasSelfReference); - return makeTextPath(name, th.hash, references.references); + [&](TextInfo ti) { + return makeTextPath(info.name, ti); }, - [&](FixedOutputHash fsh) { - return makeFixedOutputPath(name, fsh, references); + [&](FixedOutputInfo foi) { + return makeFixedOutputPath(info.name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", PathReferences { references }), hash, name); + }, info.info); } @@ -226,17 +225,14 @@ std::pair Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = h, - }), h); + return std::make_pair(makeFixedOutputPath(name, FixedOutputInfo { method, h, {} }), h); } StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { hashString(htSHA256, s), references }); } @@ -491,7 +487,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store } if (info->ca) - jsonPath.attr("ca", renderContentAddress(info->ca)); + jsonPath.attr("ca", renderMiniContentAddress(info->ca)); std::pair closureSizes; @@ -603,7 +599,7 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -681,7 +677,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -709,7 +705,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -827,19 +823,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } +std::optional ValidPathInfo::fullContentAddressOpt() const +{ + if (! ca) + return std::nullopt; + + return FullContentAddress { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](TextHash th) { + TextInfo info { th }; + assert(!hasSelfReference); + info.references = references; + return std::variant { info }; + }, + [&](FixedOutputHash foh) { + FixedOutputInfo info { foh }; + info.references = static_cast>(*this); + return std::variant { info }; + }, + }, *ca), + }; +} + bool ValidPathInfo::isContentAddressed(const Store & store) const { - if (! ca) return false; + auto fullCaOpt = fullContentAddressOpt(); - auto caPath = std::visit(overloaded { - [&](TextHash th) { - assert(!hasSelfReference); - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](FixedOutputHash fsh) { - return store.makeFixedOutputPath(path.name(), fsh, *this); - } - }, *ca); + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -879,13 +893,19 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - std::string_view name, - ContentAddress && ca, - PathReferences && refs) - : PathReferences(std::move(refs)) - , path(store.makeFixedOutputPathFromCA(name, ca, *this)) - , ca(std::move(ca)) + FullContentAddress && info) + : path(store.makeFixedOutputPathFromCA(info)) { + std::visit(overloaded { + [this](TextInfo ti) { + this->references = ti.references; + this->ca = TextHash { std::move(ti) }; + }, + [this](FixedOutputInfo foi) { + *(static_cast *>(this)) = foi.references; + this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + }, + }, std::move(info.info)); } } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index eaff7477c8c..2e1e6d64354 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -250,20 +250,11 @@ public: StorePath makeOutputPath(const string & id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath( - std::string_view name, - const FixedOutputHash & hash, - const PathReferences & references = {}) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath( - std::string_view name, - const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA( - std::string_view name, - ContentAddress ca, - const PathReferences & references = {}) const; + StorePath makeFixedOutputPathFromCA(const FullContentAddress & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -352,7 +343,8 @@ public: sizes) of a map of paths to their optional ca values. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, + virtual void querySubstitutablePathInfos(const StorePathSet & paths, + const std::set & caPaths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ @@ -754,6 +746,6 @@ std::optional decodeValidPathInfo( /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); -std::optional getDerivationCA(const BasicDerivation & drv); +std::optional getDerivationCA(const BasicDerivation & drv); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 5a3e33c0894..14706a79c2d 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -69,9 +69,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -StorePathCAMap readStorePathCAMap(const Store & store, Source & from); +std::set readFullCaSet(const Store & store, Source & from); -void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths); +void writeFullCaSet(const Store & store, Sink & out, const std::set & paths); } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index a39de041fc8..0480b357d07 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -104,7 +104,7 @@ class hintformat public: hintformat(const string &format) :fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit ^ boost::io::too_few_args_bit); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 64f2f956a8e..c6042d38478 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -158,9 +158,10 @@ static int _main(int argc, char * * argv) if (args.size() == 2) { expectedHash = Hash(args[1], ht); const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = expectedHash, + storePath = store->makeFixedOutputPath(name, FixedOutputInfo { + method, + expectedHash, + {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; @@ -218,9 +219,10 @@ static int _main(int argc, char * * argv) into the Nix store. */ storePath = store->addToStore(name, tmpFile, method, ht); - assert(*storePath == store->makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = expectedHash, + assert(*storePath == store->makeFixedOutputPath(name, FixedOutputInfo { + method, + expectedHash, + {} })); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 43f83b4d2a5..15f28382e22 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -208,9 +208,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputHash { - .method = recursive, - .hash = Hash { hash, hashAlgo }, + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + recursive, + Hash { hash, hashAlgo }, + {}, }))); } @@ -867,7 +868,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderMiniContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { } } @@ -955,7 +956,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseContentAddressOpt(readString(in)); + info.ca = parseMiniContentAddressOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index fce7d3510ff..38303270358 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -67,16 +67,19 @@ struct CmdAddToStore : MixDryRun, StoreCommand } } - ValidPathInfo info(store->makeFixedOutputPath(*namePart, FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - })); + ValidPathInfo info { + *store, + FullContentAddress { + .name = *namePart, + .info = FixedOutputInfo { + std::move(ingestionMethod), + std::move(hash), + {}, + }, + }, + }; info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index f5fbca8e724..313bca94e90 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -6,9 +6,9 @@ using namespace nix; -struct CmdMakeContentAddressable : StorePathsCommand, MixJSON +struct CmdMakeMiniContentAddressable : StorePathsCommand, MixJSON { - CmdMakeContentAddressable() + CmdMakeMiniContentAddressable() { realiseMode = Build; } @@ -75,12 +75,14 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info { *store, - path.name(), - FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narHash, + FullContentAddress { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + FileIngestionMethod::Recursive, + narHash, + std::move(refs), + }, }, - std::move(refs), }; info.narHash = narHash; info.narSize = sink.s->size(); @@ -104,4 +106,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand("make-content-addressable"); +static auto r1 = registerCommand("make-content-addressable"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index b89a44f83cc..6a52513e62b 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); - if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); + if (info->ca) ss.push_back("ca:" + renderMiniContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); std::cout << concatStringsSep(" ", ss); } From c8cd66fb66d63bf885c6ba7e450c52cec0d37dea Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 26 Jun 2020 17:22:28 -0400 Subject: [PATCH 067/102] Fix syntax --- src/libstore/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 66ce8c2e26a..08d6c707ee6 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -612,7 +612,7 @@ static void performOp(TunnelLogger * logger, ref store, SubstitutablePathInfos infos; auto paths = readStorePaths(*store, from); std::set caPaths; - if (GET_PROTOCOL_MINOR(clientVersion) > 22) { + if (GET_PROTOCOL_MINOR(clientVersion) > 22) caPaths = readFullCaSet(*store, from); logger->startWork(); store->querySubstitutablePathInfos(paths, caPaths, infos); From bff0743b29bcc999b5ed30cfab3eb1b4ab082409 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 26 Jun 2020 17:30:32 -0400 Subject: [PATCH 068/102] Add order for FullContentAddress to put in a set, we need an ordering --- src/libstore/content-address.hh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index b10943da228..8b56a81ac86 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -122,6 +122,12 @@ struct FullContentAddress { TextInfo, FixedOutputInfo > info; + + bool operator < (const FullContentAddress & other) const + { + return name < other.name; + } + }; } From 1db393fac79dc83f031763af237a1d20f2728301 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 29 Jun 2020 13:14:26 -0400 Subject: [PATCH 069/102] Fix referencesPossiblyToSelf --- src/libstore/content-address.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 8b56a81ac86..08b323729da 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -74,10 +74,10 @@ struct PathReferences template StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const { - StorePathSet references { references }; + StorePathSet refs { references }; if (hasSelfReference) - references.insert(self); - return references; + refs.insert(self); + return refs; } template From ebba7f39d0ac48921131d393f5550f5f4bc8efaa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 19:58:31 +0000 Subject: [PATCH 070/102] Fix nar info parsing --- src/libstore/nar-info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 7005b26ec74..03ddcb70075 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -57,7 +57,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) corrupt(); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") From 83f743b11d2048c464c160edc9b5bd8832ff25ba Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Jun 2020 22:17:01 +0000 Subject: [PATCH 071/102] Fix perl bindings --- perl/lib/Nix/Store.xs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 945ed49c754..3af628e1bd0 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -287,7 +287,11 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { Hash h(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + method, + h, + {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); From 5a8e8fa8b008d8db4e86d046fcbd450d6ea08c79 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Jul 2020 01:16:29 +0000 Subject: [PATCH 072/102] Use some named designators again with braces for base class --- perl/lib/Nix/Store.xs | 6 ++++-- src/libexpr/primops.cc | 12 ++++++++---- src/libfetchers/tarball.cc | 6 ++++-- src/libfetchers/tree-info.cc | 6 ++++-- src/libstore/binary-cache-store.cc | 6 ++++-- src/libstore/build.cc | 12 ++++++++---- src/libstore/local-store.cc | 6 ++++-- src/libstore/store-api.cc | 14 ++++++++++++-- src/nix-prefetch-url/nix-prefetch-url.cc | 12 ++++++++---- src/nix-store/nix-store.cc | 6 ++++-- src/nix/add-to-store.cc | 6 ++++-- src/nix/make-content-addressable.cc | 6 ++++-- 12 files changed, 68 insertions(+), 30 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 3af628e1bd0..4671b9d5439 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -288,8 +288,10 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) Hash h(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { - method, - h, + { + .method = method, + .hash = h, + }, {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3e220386b91..9ed4dbb652a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -773,8 +773,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * Hash h = newHashAllowEmpty(*outputHash, ht); auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { - ingestionMethod, - h, + { + .method = ingestionMethod, + .hash = h, + }, {}, }); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); @@ -1158,8 +1160,10 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - method, - expectedHash, + { + .method = method, + .hash = expectedHash, + }, {}, }); Path dstPath; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 197ad642853..b508fc2bbb6 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -72,8 +72,10 @@ DownloadFileResult downloadFile( FullContentAddress { .name = name, .info = FixedOutputInfo { - FileIngestionMethod::Flat, - hash, + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, {}, }, }, diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index f77b95dea0e..71c95a2f614 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -9,8 +9,10 @@ StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); return store.makeFixedOutputPath("source", FixedOutputInfo { - FileIngestionMethod::Recursive, - narHash, + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, {}, }); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 8c9bfccb7bf..3512859f684 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -348,8 +348,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath } ValidPathInfo info(makeFixedOutputPath(name, FixedOutputInfo { - method, - h, + { + .method = method, + .hash = h, + }, {}, })); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 70d429a8242..78482a3bf7b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3737,8 +3737,10 @@ void DerivationGoal::registerOutputs() : hashFile(*i.second.hash->hash.type, actualPath); auto dest = worker.store.makeFixedOutputPath(i.second.path.name(), FixedOutputInfo { - i.second.hash->method, - h2, + { + .method = i.second.hash->method, + .hash = h2, + }, {}, // TODO references }); @@ -3774,8 +3776,10 @@ void DerivationGoal::registerOutputs() ca = FullContentAddress { .name = std::string { i.second.path.name() }, .info = FixedOutputInfo { - i.second.hash->method, - h2, + { + .method = i.second.hash->method, + .hash = h2, + }, {}, }, }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b56bab20187..aa9fba27a71 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1058,8 +1058,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam Hash h = hashString(hashAlgo, dump); auto dstPath = makeFixedOutputPath(name, FixedOutputInfo { - method, - h, + { + .method = method, + .hash = h, + }, {}, }); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 91767e55f32..74b25e4aa81 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -225,14 +225,24 @@ std::pair Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(name, FixedOutputInfo { method, h, {} }), h); + FixedOutputInfo caInfo { + { + .method = method, + .hash = h, + }, + {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, TextInfo { hashString(htSHA256, s), references }); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index c6042d38478..a83437a3efe 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -159,8 +159,10 @@ static int _main(int argc, char * * argv) expectedHash = Hash(args[1], ht); const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; storePath = store->makeFixedOutputPath(name, FixedOutputInfo { - method, - expectedHash, + { + .method = method, + .hash = expectedHash, + }, {}, }); if (store->isValidPath(*storePath)) @@ -220,8 +222,10 @@ static int _main(int argc, char * * argv) storePath = store->addToStore(name, tmpFile, method, ht); assert(*storePath == store->makeFixedOutputPath(name, FixedOutputInfo { - method, - expectedHash, + { + .method = method, + .hash = expectedHash, + }, {} })); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 15f28382e22..ee031606014 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -209,8 +209,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - recursive, - Hash { hash, hashAlgo }, + { + .method = recursive, + .hash = Hash { hash, hashAlgo }, + }, {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 38303270358..c59cdcc1d6a 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -72,8 +72,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand FullContentAddress { .name = *namePart, .info = FixedOutputInfo { - std::move(ingestionMethod), - std::move(hash), + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, {}, }, }, diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 313bca94e90..1ab77c1d2bb 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -78,8 +78,10 @@ struct CmdMakeMiniContentAddressable : StorePathsCommand, MixJSON FullContentAddress { .name = std::string { path.name() }, .info = FixedOutputInfo { - FileIngestionMethod::Recursive, - narHash, + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, std::move(refs), }, }, From 7dacd78b0b29622458e1a8c694ccfd2751204d44 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Jul 2020 05:10:17 +0000 Subject: [PATCH 073/102] Fix more initialization warnings --- src/libstore/local-store.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index aa9fba27a71..5e68e5ab344 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -860,8 +860,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const s auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); infos.insert_or_assign(localPath, SubstitutablePathInfo { - info->references, - info->hasSelfReference, + { *info }, info->deriver, narInfo ? narInfo->fileSize : 0, info->narSize, @@ -1140,7 +1139,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, TextInfo { hash, references }); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); From b2e0ec825d6d6ef739ecd8d4920fbdd4d7c31059 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Jul 2020 05:10:34 +0000 Subject: [PATCH 074/102] Try to avoid include problem in perl --- src/libstore/path.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/path.hh b/src/libstore/path.hh index c82ed9f266f..c2599cf4077 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,5 +1,7 @@ #pragma once +#include + #include "types.hh" namespace nix { From c77f6c999ecb2c640da46882ee22a6f4f5b0fed3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 1 Jul 2020 17:32:07 -0400 Subject: [PATCH 075/102] Rename FullContentAddress to ContentAddress LegacyContentAddress used for all other usages --- src/libfetchers/tarball.cc | 2 +- src/libstore/build.cc | 18 +++++++++--------- src/libstore/content-address.cc | 16 ++++++++-------- src/libstore/content-address.hh | 14 +++++++------- src/libstore/daemon.cc | 8 ++++---- src/libstore/legacy-ssh-store.cc | 4 ++-- src/libstore/local-store.cc | 8 ++++---- src/libstore/local-store.hh | 2 +- src/libstore/misc.cc | 4 ++-- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 4 ++-- src/libstore/path-info.hh | 6 +++--- src/libstore/remote-store.cc | 18 +++++++++--------- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 10 +++++----- src/libstore/store-api.hh | 6 +++--- src/libstore/worker-protocol.hh | 4 ++-- src/nix-store/nix-store.cc | 4 ++-- src/nix/add-to-store.cc | 2 +- src/nix/make-content-addressable.cc | 8 ++++---- src/nix/path-info.cc | 2 +- 21 files changed, 73 insertions(+), 73 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b508fc2bbb6..c9fac531a44 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -69,7 +69,7 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { *store, - FullContentAddress { + ContentAddress { .name = name, .info = FixedOutputInfo { { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 78482a3bf7b..9cd7f92eb00 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ class Worker GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -3717,7 +3717,7 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::optional ca; + std::optional ca; if (fixedOutput) { @@ -3773,7 +3773,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = FullContentAddress { + ca = ContentAddress { .name = std::string { i.second.path.name() }, .info = FixedOutputInfo { { @@ -3847,7 +3847,7 @@ void DerivationGoal::registerOutputs() } auto info = ca - ? ValidPathInfo { worker.store, FullContentAddress { *ca } } + ? ValidPathInfo { worker.store, ContentAddress { *ca } } : ValidPathInfo { worker.store.parseStorePath(path) }; info.narHash = hash.first; info.narSize = hash.second; @@ -4281,7 +4281,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ - // TODO std::variant storePath; + // TODO std::variant storePath; StorePath storePath; /* The remaining substituters. */ @@ -4319,10 +4319,10 @@ class SubstitutionGoal : public Goal /* Content address for recomputing store path */ // TODO delete once `storePath` is variant. - std::optional ca; + std::optional ca; public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4352,7 +4352,7 @@ class SubstitutionGoal : public Goal }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) @@ -4699,7 +4699,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 477b93c5bad..a4d1c2912a2 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -21,7 +21,7 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -std::string renderMiniContentAddress(MiniContentAddress ca) { +std::string renderLegacyContentAddress(LegacyContentAddress ca) { return std::visit(overloaded { [](TextHash th) { return "text:" @@ -35,7 +35,7 @@ std::string renderMiniContentAddress(MiniContentAddress ca) { }, ca); } -MiniContentAddress parseMiniContentAddress(std::string_view rawCa) { +LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { auto prefixSeparator = rawCa.find(':'); if (prefixSeparator != string::npos) { auto prefix = string(rawCa, 0, prefixSeparator); @@ -43,7 +43,7 @@ MiniContentAddress parseMiniContentAddress(std::string_view rawCa) { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); if (*hash.type != htSHA256) { - throw Error("parseMiniContentAddress: the text hash should have type SHA256"); + throw Error("parseLegacyContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; } else if (prefix == "fixed") { @@ -62,19 +62,19 @@ MiniContentAddress parseMiniContentAddress(std::string_view rawCa) { }; } } else { - throw Error("parseMiniContentAddress: format not recognized; has to be text or fixed"); + throw Error("parseLegacyContentAddress: format not recognized; has to be text or fixed"); } } else { throw Error("Not a content address because it lacks an appropriate prefix"); } }; -std::optional parseMiniContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional {} : parseMiniContentAddress(rawCaOpt); +std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional {} : parseLegacyContentAddress(rawCaOpt); }; -std::string renderMiniContentAddress(std::optional ca) { - return ca ? renderMiniContentAddress(*ca) : ""; +std::string renderLegacyContentAddress(std::optional ca) { + return ca ? renderLegacyContentAddress(*ca) : ""; } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 08b323729da..84ff71a344f 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -40,19 +40,19 @@ struct FixedOutputHash { typedef std::variant< TextHash, // for paths computed by makeTextPath() / addTextToStore FixedOutputHash // for path computed by makeFixedOutputPath -> MiniContentAddress; +> LegacyContentAddress; /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -std::string renderMiniContentAddress(MiniContentAddress ca); +std::string renderLegacyContentAddress(LegacyContentAddress ca); -std::string renderMiniContentAddress(std::optional ca); +std::string renderLegacyContentAddress(std::optional ca); -MiniContentAddress parseMiniContentAddress(std::string_view rawCa); +LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa); -std::optional parseMiniContentAddressOpt(std::string_view rawCaOpt); +std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt); /* * References set @@ -116,14 +116,14 @@ struct FixedOutputInfo : FixedOutputHash { PathReferences references; }; -struct FullContentAddress { +struct ContentAddress { std::string name; std::variant< TextInfo, FixedOutputInfo > info; - bool operator < (const FullContentAddress & other) const + bool operator < (const ContentAddress & other) const { return name < other.name; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 08d6c707ee6..e3c599b1738 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -611,9 +611,9 @@ static void performOp(TunnelLogger * logger, ref store, case wopQuerySubstitutablePathInfos: { SubstitutablePathInfos infos; auto paths = readStorePaths(*store, from); - std::set caPaths; + std::set caPaths; if (GET_PROTOCOL_MINOR(clientVersion) > 22) - caPaths = readFullCaSet(*store, from); + caPaths = readContentAddressSet(*store, from); logger->startWork(); store->querySubstitutablePathInfos(paths, caPaths, infos); logger->stopWork(); @@ -655,7 +655,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << renderMiniContentAddress(info->ca); + << renderLegacyContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -713,7 +713,7 @@ static void performOp(TunnelLogger * logger, ref store, info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseMiniContentAddressOpt(readString(from)); + info.ca = parseLegacyContentAddressOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index e33aa46342b..a607f1022bc 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - info->ca = parseMiniContentAddressOpt(readString(conn->from)); + info->ca = parseLegacyContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } @@ -145,7 +145,7 @@ struct LegacySSHStore : public Store << info.narSize << info.ultimate << info.sigs - << renderMiniContentAddress(info.ca); + << renderLegacyContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5e68e5ab344..3086a976c30 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -592,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderMiniContentAddress(info.ca), (bool) info.ca) + (renderLegacyContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = sqlite3_last_insert_rowid(state.db); @@ -666,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = parseMiniContentAddressOpt(s); + if (s) info->ca = parseLegacyContentAddressOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -691,7 +691,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderMiniContentAddress(info.ca), (bool) info.ca) + (renderLegacyContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } @@ -845,7 +845,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 660d7a08fe8..487a27e5086 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -140,7 +140,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1097dfdd3cf..ece38fc7b2d 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -107,11 +107,11 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional getDerivationCA(const BasicDerivation & drv) +std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out != drv.outputs.end() && out->second.hash) { - return FullContentAddress { + return ContentAddress { .name = std::string { out->second.path.name() }, .info = FixedOutputInfo { *out->second.hash, {} }, }; diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 13924cc88c9..51d8d4f4b35 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseMiniContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = parseLegacyContentAddressOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (renderMiniContentAddress(info->ca)) + (renderLegacyContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 03ddcb70075..ecec5bad80b 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -70,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseMiniContentAddressOpt(value); + ca = parseLegacyContentAddressOpt(value); } pos = eol + 1; @@ -107,7 +107,7 @@ std::string NarInfo::to_string(const Store & store) const res += "Sig: " + sig + "\n"; if (ca) - res += "CA: " + renderMiniContentAddress(*ca) + "\n"; + res += "CA: " + renderLegacyContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index f0d0e6b6172..c702c60f55c 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -51,7 +51,7 @@ struct ValidPathInfo : PathReferences and the store path would be computed from the name component, ‘narHash’ and ‘references’. However, we support many types of content addresses. */ - std::optional ca; + std::optional ca; bool operator == (const ValidPathInfo & i) const { @@ -72,7 +72,7 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); - std::optional fullContentAddressOpt() const; + std::optional fullContentAddressOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -101,7 +101,7 @@ struct ValidPathInfo : PathReferences ValidPathInfo(const StorePath & path) : path(path) { }; ValidPathInfo(const Store & store, - FullContentAddress && ca); + ContentAddress && ca); virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6eb1b002a44..11bafc1ad53 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,23 +38,23 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::set readFullCaSet(const Store & store, Source & from) +std::set readContentAddressSet(const Store & store, Source & from) { - std::set paths; + std::set paths; // TODO // auto count = readNum(from); // while (count--) - // paths.insert_or_assign(store.parseStorePath(readString(from)), parseMiniContentAddressOpt(readString(from))); + // paths.insert_or_assign(store.parseStorePath(readString(from)), parseLegacyContentAddressOpt(readString(from))); return paths; } -void writeFullCaSet(const Store & store, Sink & out, const std::set & paths) +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) { // TODO //out << paths.size(); //for (auto & i : paths) { // out << store.printStorePath(i.first); - // out << renderMiniContentAddress(i.second); + // out << renderLegacyContentAddress(i.second); //} } @@ -328,7 +328,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (paths.empty() && caPaths.empty()) return; @@ -364,7 +364,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const writeStorePaths(*this, conn->to, combine()); } else { writeStorePaths(*this, conn->to, paths); - writeFullCaSet(*this, conn->to, caPaths); + writeContentAddressSet(*this, conn->to, caPaths); } conn.processStderr(); size_t count = readNum(conn->from); @@ -412,7 +412,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); - info->ca = parseMiniContentAddressOpt(readString(conn->from)); + info->ca = parseLegacyContentAddressOpt(readString(conn->from)); } } callback(std::move(info)); @@ -496,7 +496,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << renderMiniContentAddress(info.ca) + << info.ultimate << info.sigs << renderLegacyContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 7a3acd6bfa6..e30c6f2024e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -56,7 +56,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 74b25e4aa81..f3bebe3e26e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -205,7 +205,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -StorePath Store::makeFixedOutputPathFromCA(const FullContentAddress & info) const +StorePath Store::makeFixedOutputPathFromCA(const ContentAddress & info) const { // New template return std::visit(overloaded { @@ -497,7 +497,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store } if (info->ca) - jsonPath.attr("ca", renderMiniContentAddress(info->ca)); + jsonPath.attr("ca", renderLegacyContentAddress(info->ca)); std::pair closureSizes; @@ -833,12 +833,12 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullContentAddressOpt() const +std::optional ValidPathInfo::fullContentAddressOpt() const { if (! ca) return std::nullopt; - return FullContentAddress { + return ContentAddress { .name = std::string { path.name() }, .info = std::visit(overloaded { [&](TextHash th) { @@ -903,7 +903,7 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - FullContentAddress && info) + ContentAddress && info) : path(store.makeFixedOutputPathFromCA(info)) { std::visit(overloaded { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2e1e6d64354..f3c6910cc6e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -254,7 +254,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(const FullContentAddress & info) const; + StorePath makeFixedOutputPathFromCA(const ContentAddress & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -344,7 +344,7 @@ public: does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ @@ -746,6 +746,6 @@ std::optional decodeValidPathInfo( /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); -std::optional getDerivationCA(const BasicDerivation & drv); +std::optional getDerivationCA(const BasicDerivation & drv); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 14706a79c2d..a692e855cd9 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -69,9 +69,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::set readFullCaSet(const Store & store, Source & from); +std::set readContentAddressSet(const Store & store, Source & from); -void writeFullCaSet(const Store & store, Sink & out, const std::set & paths); +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index ee031606014..b96363e5667 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -870,7 +870,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderMiniContentAddress(info->ca) << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderLegacyContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { } } @@ -958,7 +958,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseMiniContentAddressOpt(readString(in)); + info.ca = parseLegacyContentAddressOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index c59cdcc1d6a..1ebe38344f2 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,7 +69,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - FullContentAddress { + ContentAddress { .name = *namePart, .info = FixedOutputInfo { { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 1ab77c1d2bb..7e3c0ee4e05 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -6,9 +6,9 @@ using namespace nix; -struct CmdMakeMiniContentAddressable : StorePathsCommand, MixJSON +struct CmdMakeContentAddressable : StorePathsCommand, MixJSON { - CmdMakeMiniContentAddressable() + CmdMakeContentAddressable() { realiseMode = Build; } @@ -75,7 +75,7 @@ struct CmdMakeMiniContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info { *store, - FullContentAddress { + ContentAddress { .name = std::string { path.name() }, .info = FixedOutputInfo { { @@ -108,4 +108,4 @@ struct CmdMakeMiniContentAddressable : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand("make-content-addressable"); +static auto r1 = registerCommand("make-content-addressable"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 6a52513e62b..126b69635e3 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); - if (info->ca) ss.push_back("ca:" + renderMiniContentAddress(*info->ca)); + if (info->ca) ss.push_back("ca:" + renderLegacyContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); std::cout << concatStringsSep(" ", ss); } From 7271778defe9bad01e6b8e705ab75c047eb24d68 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 5 Jul 2020 21:39:04 +0000 Subject: [PATCH 076/102] See if setting -std=c++17 for PERL helps --- perl/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/Makefile b/perl/Makefile index 7ddb0cf692c..259ed7dc3f1 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,6 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -g -Wall +GLOBAL_CXXFLAGS += -g -Wall -std=c++17 -include Makefile.config From ff072964402897cde7ed3cb71930c693dff6ebe6 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 13 Jul 2020 14:58:19 -0400 Subject: [PATCH 077/102] Rename ContentAddress to StorePathDescriptor --- src/libfetchers/tarball.cc | 2 +- src/libstore/build.cc | 18 +++++++++--------- src/libstore/content-address.hh | 4 ++-- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/misc.cc | 4 ++-- src/libstore/path-info.hh | 4 ++-- src/libstore/remote-store.cc | 8 ++++---- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 8 ++++---- src/libstore/store-api.hh | 6 +++--- src/libstore/worker-protocol.hh | 4 ++-- src/nix/add-to-store.cc | 2 +- src/nix/make-content-addressable.cc | 2 +- 15 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index c9fac531a44..7d4d3d6c711 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -69,7 +69,7 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = name, .info = FixedOutputInfo { { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 299887ec1fc..1df95fb3e00 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ class Worker GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -3717,7 +3717,7 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::optional ca; + std::optional ca; if (fixedOutput) { @@ -3773,7 +3773,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = ContentAddress { + ca = StorePathDescriptor { .name = std::string { i.second.path.name() }, .info = FixedOutputInfo { { @@ -3847,7 +3847,7 @@ void DerivationGoal::registerOutputs() } auto info = ca - ? ValidPathInfo { worker.store, ContentAddress { *ca } } + ? ValidPathInfo { worker.store, StorePathDescriptor { *ca } } : ValidPathInfo { worker.store.parseStorePath(path) }; info.narHash = hash.first; info.narSize = hash.second; @@ -4281,7 +4281,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ - // TODO std::variant storePath; + // TODO std::variant storePath; StorePath storePath; /* The remaining substituters. */ @@ -4319,10 +4319,10 @@ class SubstitutionGoal : public Goal /* Content address for recomputing store path */ // TODO delete once `storePath` is variant. - std::optional ca; + std::optional ca; public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4352,7 +4352,7 @@ class SubstitutionGoal : public Goal }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) @@ -4699,7 +4699,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 84ff71a344f..10a5fcf6657 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -116,14 +116,14 @@ struct FixedOutputInfo : FixedOutputHash { PathReferences references; }; -struct ContentAddress { +struct StorePathDescriptor { std::string name; std::variant< TextInfo, FixedOutputInfo > info; - bool operator < (const ContentAddress & other) const + bool operator < (const StorePathDescriptor & other) const { return name < other.name; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index cb0aa282cfb..10e842d8b74 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -620,7 +620,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopQuerySubstitutablePathInfos: { SubstitutablePathInfos infos; auto paths = readStorePaths(*store, from); - std::set caPaths; + std::set caPaths; if (GET_PROTOCOL_MINOR(clientVersion) > 22) caPaths = readContentAddressSet(*store, from); logger->startWork(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0ac1da03a5..173e97bed59 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -848,7 +848,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f835c1a9fa0..a4fbd65d5de 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -140,7 +140,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ece38fc7b2d..13823a933c3 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -107,11 +107,11 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional getDerivationCA(const BasicDerivation & drv) +std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out != drv.outputs.end() && out->second.hash) { - return ContentAddress { + return StorePathDescriptor { .name = std::string { out->second.path.name() }, .info = FixedOutputInfo { *out->second.hash, {} }, }; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index c702c60f55c..febc3e6fe65 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,7 +72,7 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); - std::optional fullContentAddressOpt() const; + std::optional fullContentAddressOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -101,7 +101,7 @@ struct ValidPathInfo : PathReferences ValidPathInfo(const StorePath & path) : path(path) { }; ValidPathInfo(const Store & store, - ContentAddress && ca); + StorePathDescriptor && ca); virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 443c9c45a7b..27de5c28456 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -39,9 +39,9 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::set readContentAddressSet(const Store & store, Source & from) +std::set readContentAddressSet(const Store & store, Source & from) { - std::set paths; + std::set paths; // TODO // auto count = readNum(from); // while (count--) @@ -49,7 +49,7 @@ std::set readContentAddressSet(const Store & store, Source & fro return paths; } -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) { // TODO //out << paths.size(); @@ -352,7 +352,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (paths.empty() && caPaths.empty()) return; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a2da5a0fc4a..be931dabab0 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -57,7 +57,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4928a16f6c5..5f7375abee2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -205,7 +205,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -StorePath Store::makeFixedOutputPathFromCA(const ContentAddress & info) const +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & info) const { // New template return std::visit(overloaded { @@ -843,12 +843,12 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullContentAddressOpt() const +std::optional ValidPathInfo::fullContentAddressOpt() const { if (! ca) return std::nullopt; - return ContentAddress { + return StorePathDescriptor { .name = std::string { path.name() }, .info = std::visit(overloaded { [&](TextHash th) { @@ -913,7 +913,7 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - ContentAddress && info) + StorePathDescriptor && info) : path(store.makeFixedOutputPathFromCA(info)) { std::visit(overloaded { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 37b75e73052..6f08151a9bf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -254,7 +254,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(const ContentAddress & info) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -347,7 +347,7 @@ public: does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ @@ -749,6 +749,6 @@ std::optional decodeValidPathInfo( /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); -std::optional getDerivationCA(const BasicDerivation & drv); +std::optional getDerivationCA(const BasicDerivation & drv); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 0b6816abb42..cdb9b9be4e6 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -70,9 +70,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::set readContentAddressSet(const Store & store, Source & from); +std::set readContentAddressSet(const Store & store, Source & from); -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 1ebe38344f2..b9aa8dd162f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,7 +69,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = *namePart, .info = FixedOutputInfo { { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 7e3c0ee4e05..028c0f1442d 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -75,7 +75,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = std::string { path.name() }, .info = FixedOutputInfo { { From ade0f1a440120475bfe4a985f66d050f6cd01279 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 14 Jul 2020 10:41:47 -0400 Subject: [PATCH 078/102] Rename functions related to ContentAddress --- src/libstore/daemon.cc | 2 +- src/libstore/path-info.hh | 2 +- src/libstore/remote-store.cc | 6 +++--- src/libstore/store-api.cc | 10 +++++----- src/libstore/worker-protocol.hh | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 10e842d8b74..5ad7f6ade49 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -622,7 +622,7 @@ static void performOp(TunnelLogger * logger, ref store, auto paths = readStorePaths(*store, from); std::set caPaths; if (GET_PROTOCOL_MINOR(clientVersion) > 22) - caPaths = readContentAddressSet(*store, from); + caPaths = readStorePathDescriptorSet(*store, from); logger->startWork(); store->querySubstitutablePathInfos(paths, caPaths, infos); logger->stopWork(); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index febc3e6fe65..5b2716b769f 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,7 +72,7 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); - std::optional fullContentAddressOpt() const; + std::optional fullStorePathDescriptorOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 27de5c28456..aee82599bf0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -39,7 +39,7 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::set readContentAddressSet(const Store & store, Source & from) +std::set readStorePathDescriptorSet(const Store & store, Source & from) { std::set paths; // TODO @@ -49,7 +49,7 @@ std::set readContentAddressSet(const Store & store, Source return paths; } -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) +void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set & paths) { // TODO //out << paths.size(); @@ -388,7 +388,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const writeStorePaths(*this, conn->to, combine()); } else { writeStorePaths(*this, conn->to, paths); - writeContentAddressSet(*this, conn->to, caPaths); + writeStorePathDescriptorSet(*this, conn->to, caPaths); } conn.processStderr(); size_t count = readNum(conn->from); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5f7375abee2..80d336209a6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -619,7 +619,7 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -697,7 +697,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -725,7 +725,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -843,7 +843,7 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullContentAddressOpt() const +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const { if (! ca) return std::nullopt; @@ -868,7 +868,7 @@ std::optional ValidPathInfo::fullContentAddressOpt() const bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = fullContentAddressOpt(); + auto fullCaOpt = fullStorePathDescriptorOpt(); if (! fullCaOpt) return false; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index cdb9b9be4e6..f42ce125f2f 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -70,9 +70,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::set readContentAddressSet(const Store & store, Source & from); +std::set readStorePathDescriptorSet(const Store & store, Source & from); -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); +void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set & paths); void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); From 08bf6c414157dc14221b760f395c0920149c5095 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 14 Jul 2020 10:46:18 -0400 Subject: [PATCH 079/102] Rename LegacyContentAddress and related functions --- src/libstore/content-address.cc | 16 ++++++++-------- src/libstore/content-address.hh | 10 +++++----- src/libstore/daemon.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 4 ++-- src/libstore/local-store.cc | 6 +++--- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 4 ++-- src/libstore/path-info.hh | 2 +- src/libstore/remote-store.cc | 8 ++++---- src/libstore/store-api.cc | 2 +- src/nix-store/nix-store.cc | 4 ++-- src/nix/path-info.cc | 2 +- 12 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a4d1c2912a2..ebd1763f522 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -21,7 +21,7 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -std::string renderLegacyContentAddress(LegacyContentAddress ca) { +std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { return "text:" @@ -35,7 +35,7 @@ std::string renderLegacyContentAddress(LegacyContentAddress ca) { }, ca); } -LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { +ContentAddress parseContentAddress(std::string_view rawCa) { auto prefixSeparator = rawCa.find(':'); if (prefixSeparator != string::npos) { auto prefix = string(rawCa, 0, prefixSeparator); @@ -43,7 +43,7 @@ LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); if (*hash.type != htSHA256) { - throw Error("parseLegacyContentAddress: the text hash should have type SHA256"); + throw Error("parseContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; } else if (prefix == "fixed") { @@ -62,19 +62,19 @@ LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { }; } } else { - throw Error("parseLegacyContentAddress: format not recognized; has to be text or fixed"); + throw Error("parseContentAddress: format not recognized; has to be text or fixed"); } } else { throw Error("Not a content address because it lacks an appropriate prefix"); } }; -std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional {} : parseLegacyContentAddress(rawCaOpt); +std::optional parseContentAddressOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional {} : parseContentAddress(rawCaOpt); }; -std::string renderLegacyContentAddress(std::optional ca) { - return ca ? renderLegacyContentAddress(*ca) : ""; +std::string renderContentAddress(std::optional ca) { + return ca ? renderContentAddress(*ca) : ""; } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 10a5fcf6657..5f18b0b9d4f 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -40,19 +40,19 @@ struct FixedOutputHash { typedef std::variant< TextHash, // for paths computed by makeTextPath() / addTextToStore FixedOutputHash // for path computed by makeFixedOutputPath -> LegacyContentAddress; +> ContentAddress; /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -std::string renderLegacyContentAddress(LegacyContentAddress ca); +std::string renderContentAddress(ContentAddress ca); -std::string renderLegacyContentAddress(std::optional ca); +std::string renderContentAddress(std::optional ca); -LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa); +ContentAddress parseContentAddress(std::string_view rawCa); -std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt); +std::optional parseContentAddressOpt(std::string_view rawCaOpt); /* * References set diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5ad7f6ade49..9a7524d39ba 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -664,7 +664,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << renderLegacyContentAddress(info->ca); + << renderContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -722,7 +722,7 @@ static void performOp(TunnelLogger * logger, ref store, info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseLegacyContentAddressOpt(readString(from)); + info.ca = parseContentAddressOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a607f1022bc..f01e642a035 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - info->ca = parseLegacyContentAddressOpt(readString(conn->from)); + info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } @@ -145,7 +145,7 @@ struct LegacySSHStore : public Store << info.narSize << info.ultimate << info.sigs - << renderLegacyContentAddress(info.ca); + << renderContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 173e97bed59..a73966df306 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -592,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderLegacyContentAddress(info.ca), (bool) info.ca) + (renderContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = sqlite3_last_insert_rowid(state.db); @@ -666,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = parseLegacyContentAddressOpt(s); + if (s) info->ca = parseContentAddressOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -691,7 +691,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderLegacyContentAddress(info.ca), (bool) info.ca) + (renderContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 51d8d4f4b35..c543f6ea297 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseLegacyContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (renderLegacyContentAddress(info->ca)) + (renderContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ecec5bad80b..0796de46648 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -70,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseLegacyContentAddressOpt(value); + ca = parseContentAddressOpt(value); } pos = eol + 1; @@ -107,7 +107,7 @@ std::string NarInfo::to_string(const Store & store) const res += "Sig: " + sig + "\n"; if (ca) - res += "CA: " + renderLegacyContentAddress(*ca) + "\n"; + res += "CA: " + renderContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 5b2716b769f..fcd080933fb 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -51,7 +51,7 @@ struct ValidPathInfo : PathReferences and the store path would be computed from the name component, ‘narHash’ and ‘references’. However, we support many types of content addresses. */ - std::optional ca; + std::optional ca; bool operator == (const ValidPathInfo & i) const { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index aee82599bf0..100feeafe8d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -45,7 +45,7 @@ std::set readStorePathDescriptorSet(const Store & store, So // TODO // auto count = readNum(from); // while (count--) - // paths.insert_or_assign(store.parseStorePath(readString(from)), parseLegacyContentAddressOpt(readString(from))); + // paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); return paths; } @@ -55,7 +55,7 @@ void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set //out << paths.size(); //for (auto & i : paths) { // out << store.printStorePath(i.first); - // out << renderLegacyContentAddress(i.second); + // out << renderContentAddress(i.second); //} } @@ -436,7 +436,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); - info->ca = parseLegacyContentAddressOpt(readString(conn->from)); + info->ca = parseContentAddressOpt(readString(conn->from)); } } callback(std::move(info)); @@ -532,7 +532,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << renderLegacyContentAddress(info.ca) + << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 80d336209a6..d92537845d8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -507,7 +507,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store } if (info->ca) - jsonPath.attr("ca", renderLegacyContentAddress(info->ca)); + jsonPath.attr("ca", renderContentAddress(info->ca)); std::pair closureSizes; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b96363e5667..04e408c73fa 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -870,7 +870,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderLegacyContentAddress(info->ca) << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { } } @@ -958,7 +958,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseLegacyContentAddressOpt(readString(in)); + info.ca = parseContentAddressOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 126b69635e3..b89a44f83cc 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); - if (info->ca) ss.push_back("ca:" + renderLegacyContentAddress(*info->ca)); + if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); std::cout << concatStringsSep(" ", ss); } From c07cecca930e46871be52e5f5c0d496cd351ef7e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Jul 2020 21:11:08 +0000 Subject: [PATCH 080/102] TEMP --- src/libstore/local-store.cc | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 44254d4e9f9..00e3db9cf94 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1111,15 +1111,8 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); - /* For computing the NAR hash. */ - auto sha256Sink = std::make_unique(htSHA256); - - /* For computing the store path. In recursive SHA-256 mode, this - is the same as the NAR hash, so no need to do it again. */ - std::unique_ptr hashSink = - method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 - ? nullptr - : std::make_unique(hashAlgo); + /* For computing the store path. */ + auto hashSink = std::make_unique(hashAlgo); /* Read the source path into memory, but only if it's up to narBufferSize bytes. If it's larger, write it to a temporary @@ -1128,13 +1121,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, temporary path. Otherwise, we move it to the destination store path. */ bool inMemory = true; - std::string nar; + std::string nar; // TODO rename from "nar" to "dump" auto source = sinkToSource([&](Sink & sink) { LambdaSink sink2([&](const unsigned char * buf, size_t len) { - (*sha256Sink)(buf, len); - if (hashSink) (*hashSink)(buf, len); + (*hashSink)(buf, len); if (inMemory) { if (nar.size() + len > settings.narBufferSize) { @@ -1204,9 +1196,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, /* The NAR fits in memory, so we didn't do restorePath(). */ } - auto sha256 = sha256Sink->finish(); - - Hash hash = hashSink ? hashSink->finish().first : sha256.first; + auto [hash, size] = hashSink->finish(); auto dstPath = makeFixedOutputPath(method, hash, name); @@ -1249,13 +1239,22 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, throw Error("renaming '%s' to '%s'", tempPath, realPath); } + /* For computing the nar hash. In recursive SHA-256 mode, this + is the same as the store hash, so no need to do it again. */ + auto narHash = std::pair { hash, size }; + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { + HashSink narSink { htSHA256 }; + dumpPath(realPath, narSink); + narHash = narSink.finish(); + } + canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath optimisePath(realPath); ValidPathInfo info(dstPath); - info.narHash = sha256.first; - info.narSize = sha256.second; + info.narHash = narHash.first; + info.narSize = narHash.second; info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } From d0eaf2bc51c82e5a72cd6bf2f035156c85b083aa Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 13 Jul 2020 14:58:19 -0400 Subject: [PATCH 081/102] Rename ContentAddress to StorePathDescriptor (cherry picked from commit ff072964402897cde7ed3cb71930c693dff6ebe6) --- src/libfetchers/tarball.cc | 2 +- src/libstore/build.cc | 18 +++++++++--------- src/libstore/content-address.hh | 4 ++-- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/misc.cc | 4 ++-- src/libstore/path-info.hh | 4 ++-- src/libstore/remote-store.cc | 8 ++++---- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 8 ++++---- src/libstore/store-api.hh | 6 +++--- src/libstore/worker-protocol.hh | 4 ++-- src/nix/add-to-store.cc | 2 +- src/nix/make-content-addressable.cc | 2 +- 15 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index c9fac531a44..7d4d3d6c711 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -69,7 +69,7 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = name, .info = FixedOutputInfo { { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 299887ec1fc..1df95fb3e00 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ class Worker GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -3717,7 +3717,7 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::optional ca; + std::optional ca; if (fixedOutput) { @@ -3773,7 +3773,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = ContentAddress { + ca = StorePathDescriptor { .name = std::string { i.second.path.name() }, .info = FixedOutputInfo { { @@ -3847,7 +3847,7 @@ void DerivationGoal::registerOutputs() } auto info = ca - ? ValidPathInfo { worker.store, ContentAddress { *ca } } + ? ValidPathInfo { worker.store, StorePathDescriptor { *ca } } : ValidPathInfo { worker.store.parseStorePath(path) }; info.narHash = hash.first; info.narSize = hash.second; @@ -4281,7 +4281,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ - // TODO std::variant storePath; + // TODO std::variant storePath; StorePath storePath; /* The remaining substituters. */ @@ -4319,10 +4319,10 @@ class SubstitutionGoal : public Goal /* Content address for recomputing store path */ // TODO delete once `storePath` is variant. - std::optional ca; + std::optional ca; public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4352,7 +4352,7 @@ class SubstitutionGoal : public Goal }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) @@ -4699,7 +4699,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 84ff71a344f..10a5fcf6657 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -116,14 +116,14 @@ struct FixedOutputInfo : FixedOutputHash { PathReferences references; }; -struct ContentAddress { +struct StorePathDescriptor { std::string name; std::variant< TextInfo, FixedOutputInfo > info; - bool operator < (const ContentAddress & other) const + bool operator < (const StorePathDescriptor & other) const { return name < other.name; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e26fc9f1c16..f56576818d9 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -620,7 +620,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopQuerySubstitutablePathInfos: { SubstitutablePathInfos infos; auto paths = readStorePaths(*store, from); - std::set caPaths; + std::set caPaths; if (GET_PROTOCOL_MINOR(clientVersion) > 22) caPaths = readContentAddressSet(*store, from); logger->startWork(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0ac1da03a5..173e97bed59 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -848,7 +848,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f835c1a9fa0..a4fbd65d5de 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -140,7 +140,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ece38fc7b2d..13823a933c3 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -107,11 +107,11 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional getDerivationCA(const BasicDerivation & drv) +std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out != drv.outputs.end() && out->second.hash) { - return ContentAddress { + return StorePathDescriptor { .name = std::string { out->second.path.name() }, .info = FixedOutputInfo { *out->second.hash, {} }, }; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index c702c60f55c..febc3e6fe65 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,7 +72,7 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); - std::optional fullContentAddressOpt() const; + std::optional fullContentAddressOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -101,7 +101,7 @@ struct ValidPathInfo : PathReferences ValidPathInfo(const StorePath & path) : path(path) { }; ValidPathInfo(const Store & store, - ContentAddress && ca); + StorePathDescriptor && ca); virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8e2a9dac279..bb2f49a3b26 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,9 +38,9 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::set readContentAddressSet(const Store & store, Source & from) +std::set readContentAddressSet(const Store & store, Source & from) { - std::set paths; + std::set paths; // TODO // auto count = readNum(from); // while (count--) @@ -48,7 +48,7 @@ std::set readContentAddressSet(const Store & store, Source & fro return paths; } -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) { // TODO //out << paths.size(); @@ -351,7 +351,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (paths.empty() && caPaths.empty()) return; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a2da5a0fc4a..be931dabab0 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -57,7 +57,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4928a16f6c5..5f7375abee2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -205,7 +205,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -StorePath Store::makeFixedOutputPathFromCA(const ContentAddress & info) const +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & info) const { // New template return std::visit(overloaded { @@ -843,12 +843,12 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullContentAddressOpt() const +std::optional ValidPathInfo::fullContentAddressOpt() const { if (! ca) return std::nullopt; - return ContentAddress { + return StorePathDescriptor { .name = std::string { path.name() }, .info = std::visit(overloaded { [&](TextHash th) { @@ -913,7 +913,7 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - ContentAddress && info) + StorePathDescriptor && info) : path(store.makeFixedOutputPathFromCA(info)) { std::visit(overloaded { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 37b75e73052..6f08151a9bf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -254,7 +254,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(const ContentAddress & info) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -347,7 +347,7 @@ public: does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ @@ -749,6 +749,6 @@ std::optional decodeValidPathInfo( /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); -std::optional getDerivationCA(const BasicDerivation & drv); +std::optional getDerivationCA(const BasicDerivation & drv); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 0b6816abb42..cdb9b9be4e6 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -70,9 +70,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::set readContentAddressSet(const Store & store, Source & from); +std::set readContentAddressSet(const Store & store, Source & from); -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 1ebe38344f2..b9aa8dd162f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,7 +69,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = *namePart, .info = FixedOutputInfo { { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 7e3c0ee4e05..028c0f1442d 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -75,7 +75,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = std::string { path.name() }, .info = FixedOutputInfo { { From 1cf131f78a97a1050e9a1de1f2dfc67f7ce4014d Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 14 Jul 2020 10:41:47 -0400 Subject: [PATCH 082/102] Rename functions related to ContentAddress (cherry picked from commit ade0f1a440120475bfe4a985f66d050f6cd01279) --- src/libstore/daemon.cc | 2 +- src/libstore/path-info.hh | 2 +- src/libstore/remote-store.cc | 6 +++--- src/libstore/store-api.cc | 10 +++++----- src/libstore/worker-protocol.hh | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f56576818d9..97666f06a56 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -622,7 +622,7 @@ static void performOp(TunnelLogger * logger, ref store, auto paths = readStorePaths(*store, from); std::set caPaths; if (GET_PROTOCOL_MINOR(clientVersion) > 22) - caPaths = readContentAddressSet(*store, from); + caPaths = readStorePathDescriptorSet(*store, from); logger->startWork(); store->querySubstitutablePathInfos(paths, caPaths, infos); logger->stopWork(); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index febc3e6fe65..5b2716b769f 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,7 +72,7 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); - std::optional fullContentAddressOpt() const; + std::optional fullStorePathDescriptorOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bb2f49a3b26..f8687f6bdf3 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,7 +38,7 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::set readContentAddressSet(const Store & store, Source & from) +std::set readStorePathDescriptorSet(const Store & store, Source & from) { std::set paths; // TODO @@ -48,7 +48,7 @@ std::set readContentAddressSet(const Store & store, Source return paths; } -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) +void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set & paths) { // TODO //out << paths.size(); @@ -387,7 +387,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const writeStorePaths(*this, conn->to, combine()); } else { writeStorePaths(*this, conn->to, paths); - writeContentAddressSet(*this, conn->to, caPaths); + writeStorePathDescriptorSet(*this, conn->to, caPaths); } conn.processStderr(); size_t count = readNum(conn->from); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5f7375abee2..80d336209a6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -619,7 +619,7 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -697,7 +697,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -725,7 +725,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -843,7 +843,7 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullContentAddressOpt() const +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const { if (! ca) return std::nullopt; @@ -868,7 +868,7 @@ std::optional ValidPathInfo::fullContentAddressOpt() const bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = fullContentAddressOpt(); + auto fullCaOpt = fullStorePathDescriptorOpt(); if (! fullCaOpt) return false; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index cdb9b9be4e6..f42ce125f2f 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -70,9 +70,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::set readContentAddressSet(const Store & store, Source & from); +std::set readStorePathDescriptorSet(const Store & store, Source & from); -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); +void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set & paths); void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); From 74de4c79bfded18508c3d1b3c5f64c21d5182c06 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 14 Jul 2020 10:46:18 -0400 Subject: [PATCH 083/102] Rename LegacyContentAddress and related functions (cherry picked from commit 08bf6c414157dc14221b760f395c0920149c5095) --- src/libstore/content-address.cc | 16 ++++++++-------- src/libstore/content-address.hh | 10 +++++----- src/libstore/daemon.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 4 ++-- src/libstore/local-store.cc | 6 +++--- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 4 ++-- src/libstore/path-info.hh | 2 +- src/libstore/remote-store.cc | 8 ++++---- src/libstore/store-api.cc | 2 +- src/nix-store/nix-store.cc | 4 ++-- src/nix/path-info.cc | 2 +- 12 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a4d1c2912a2..ebd1763f522 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -21,7 +21,7 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -std::string renderLegacyContentAddress(LegacyContentAddress ca) { +std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { return "text:" @@ -35,7 +35,7 @@ std::string renderLegacyContentAddress(LegacyContentAddress ca) { }, ca); } -LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { +ContentAddress parseContentAddress(std::string_view rawCa) { auto prefixSeparator = rawCa.find(':'); if (prefixSeparator != string::npos) { auto prefix = string(rawCa, 0, prefixSeparator); @@ -43,7 +43,7 @@ LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); if (*hash.type != htSHA256) { - throw Error("parseLegacyContentAddress: the text hash should have type SHA256"); + throw Error("parseContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; } else if (prefix == "fixed") { @@ -62,19 +62,19 @@ LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { }; } } else { - throw Error("parseLegacyContentAddress: format not recognized; has to be text or fixed"); + throw Error("parseContentAddress: format not recognized; has to be text or fixed"); } } else { throw Error("Not a content address because it lacks an appropriate prefix"); } }; -std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional {} : parseLegacyContentAddress(rawCaOpt); +std::optional parseContentAddressOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional {} : parseContentAddress(rawCaOpt); }; -std::string renderLegacyContentAddress(std::optional ca) { - return ca ? renderLegacyContentAddress(*ca) : ""; +std::string renderContentAddress(std::optional ca) { + return ca ? renderContentAddress(*ca) : ""; } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 10a5fcf6657..5f18b0b9d4f 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -40,19 +40,19 @@ struct FixedOutputHash { typedef std::variant< TextHash, // for paths computed by makeTextPath() / addTextToStore FixedOutputHash // for path computed by makeFixedOutputPath -> LegacyContentAddress; +> ContentAddress; /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -std::string renderLegacyContentAddress(LegacyContentAddress ca); +std::string renderContentAddress(ContentAddress ca); -std::string renderLegacyContentAddress(std::optional ca); +std::string renderContentAddress(std::optional ca); -LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa); +ContentAddress parseContentAddress(std::string_view rawCa); -std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt); +std::optional parseContentAddressOpt(std::string_view rawCaOpt); /* * References set diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 97666f06a56..767397d1055 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -664,7 +664,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << renderLegacyContentAddress(info->ca); + << renderContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -722,7 +722,7 @@ static void performOp(TunnelLogger * logger, ref store, info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseLegacyContentAddressOpt(readString(from)); + info.ca = parseContentAddressOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a607f1022bc..f01e642a035 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - info->ca = parseLegacyContentAddressOpt(readString(conn->from)); + info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } @@ -145,7 +145,7 @@ struct LegacySSHStore : public Store << info.narSize << info.ultimate << info.sigs - << renderLegacyContentAddress(info.ca); + << renderContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 173e97bed59..a73966df306 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -592,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderLegacyContentAddress(info.ca), (bool) info.ca) + (renderContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = sqlite3_last_insert_rowid(state.db); @@ -666,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = parseLegacyContentAddressOpt(s); + if (s) info->ca = parseContentAddressOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -691,7 +691,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderLegacyContentAddress(info.ca), (bool) info.ca) + (renderContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 51d8d4f4b35..c543f6ea297 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseLegacyContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (renderLegacyContentAddress(info->ca)) + (renderContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ecec5bad80b..0796de46648 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -70,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseLegacyContentAddressOpt(value); + ca = parseContentAddressOpt(value); } pos = eol + 1; @@ -107,7 +107,7 @@ std::string NarInfo::to_string(const Store & store) const res += "Sig: " + sig + "\n"; if (ca) - res += "CA: " + renderLegacyContentAddress(*ca) + "\n"; + res += "CA: " + renderContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 5b2716b769f..fcd080933fb 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -51,7 +51,7 @@ struct ValidPathInfo : PathReferences and the store path would be computed from the name component, ‘narHash’ and ‘references’. However, we support many types of content addresses. */ - std::optional ca; + std::optional ca; bool operator == (const ValidPathInfo & i) const { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f8687f6bdf3..57ccba32f1e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -44,7 +44,7 @@ std::set readStorePathDescriptorSet(const Store & store, So // TODO // auto count = readNum(from); // while (count--) - // paths.insert_or_assign(store.parseStorePath(readString(from)), parseLegacyContentAddressOpt(readString(from))); + // paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); return paths; } @@ -54,7 +54,7 @@ void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set //out << paths.size(); //for (auto & i : paths) { // out << store.printStorePath(i.first); - // out << renderLegacyContentAddress(i.second); + // out << renderContentAddress(i.second); //} } @@ -435,7 +435,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); - info->ca = parseLegacyContentAddressOpt(readString(conn->from)); + info->ca = parseContentAddressOpt(readString(conn->from)); } } callback(std::move(info)); @@ -531,7 +531,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << renderLegacyContentAddress(info.ca) + << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 80d336209a6..d92537845d8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -507,7 +507,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store } if (info->ca) - jsonPath.attr("ca", renderLegacyContentAddress(info->ca)); + jsonPath.attr("ca", renderContentAddress(info->ca)); std::pair closureSizes; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b96363e5667..04e408c73fa 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -870,7 +870,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderLegacyContentAddress(info->ca) << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { } } @@ -958,7 +958,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseLegacyContentAddressOpt(readString(in)); + info.ca = parseContentAddressOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 126b69635e3..b89a44f83cc 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); - if (info->ca) ss.push_back("ca:" + renderLegacyContentAddress(*info->ca)); + if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); std::cout << concatStringsSep(" ", ss); } From ecf65a6aab9f15776fa00ac99e8b87fa75ffac38 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Jul 2020 19:35:13 +0000 Subject: [PATCH 084/102] Use StorePathDescriptor more widely --- src/build-remote/build-remote.cc | 7 +- src/libexpr/primops.cc | 8 +- src/libexpr/primops/context.cc | 6 +- src/libexpr/primops/fetchTree.cc | 14 ++ src/libexpr/value-to-xml.hh | 2 +- src/libfetchers/fetchers.cc | 42 +++++- src/libfetchers/fetchers.hh | 4 + src/libfetchers/tree-info.hh | 4 + src/libstore/binary-cache-store.cc | 15 ++- src/libstore/binary-cache-store.hh | 8 +- src/libstore/build.cc | 76 ++++++----- src/libstore/content-address.cc | 161 ++++++++++++++++++---- src/libstore/content-address.hh | 27 +++- src/libstore/daemon.cc | 27 ++-- src/libstore/export-import.cc | 12 +- src/libstore/globals.cc | 13 +- src/libstore/globals.hh | 4 + src/libstore/legacy-ssh-store.cc | 8 +- src/libstore/local-fs-store.cc | 9 +- src/libstore/local-store.cc | 184 +++++++++----------------- src/libstore/local-store.hh | 8 +- src/libstore/misc.cc | 3 +- src/libstore/nar-info.hh | 2 + src/libstore/remote-store.cc | 9 +- src/libstore/remote-store.hh | 6 +- src/libstore/s3-binary-cache-store.cc | 3 +- src/libstore/ssh-store.cc | 5 +- src/libstore/store-api.cc | 97 +++++++++----- src/libstore/store-api.hh | 30 +++-- src/libutil/archive.hh | 8 -- src/libutil/error.hh | 3 +- src/libutil/serialise.cc | 17 +++ src/libutil/serialise.hh | 17 ++- src/libutil/types.hh | 1 + src/libutil/util.hh | 5 + src/nix-env/nix-env.cc | 3 +- src/nix-env/user-env.cc | 3 +- src/nix-store/nix-store.cc | 15 ++- src/nix/command.cc | 3 +- src/nix/repl.cc | 3 +- src/nix/upgrade-nix.cc | 3 +- src/nix/verify.cc | 6 +- 42 files changed, 572 insertions(+), 309 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index e071174960a..26c7d3b5863 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -253,8 +253,11 @@ static int _main(int argc, char * * argv) throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); StorePathSet missing; - for (auto & path : outputs) - if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path)); + for (auto & path : outputs) { + auto storePath = store->parseStorePath(path); + if (!store->isValidPath(storePath)) + missing.insert(store->parseStorePath(path)); + } if (!missing.empty()) { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d1ad2f7db30..92110d45019 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -103,7 +103,13 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Path realPath = state.checkSourcePath(state.toRealPath(path, context)); // FIXME - if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) { + auto complicatedCondition = [&]() -> bool { + if (!state.store->isStorePath(path)) + return false; + auto path2 = state.store->parseStorePath(path); + return state.store->isValidPath(path2) && isDerivation(path); + }; + if (complicatedCondition()) { Derivation drv = readDerivation(*state.store, realPath); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index dbb93bae6a2..c0eb1b7c6cd 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -150,8 +150,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg .hint = hintfmt("Context key '%s' is not a store path", i.name), .errPos = *i.pos }); - if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(i.name)); + if (!settings.readOnlyMode) { + auto path = state.store->parseStorePath(i.name); + state.store->ensurePath(path); + } state.forceAttrs(*i.value, *i.pos); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f6d8cca44b2..3a131f2d20f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -134,6 +134,20 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + // try to substitute if we can + if (settings.useSubstitutes && expectedHash) { + auto substitutableStorePath = fetchers::trySubstitute(state.store, + unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, *expectedHash, name); + if (substitutableStorePath) { + auto substitutablePath = state.store->toRealPath(*substitutableStorePath); + if (state.allowedPaths) + state.allowedPaths->insert(substitutablePath); + + mkString(v, substitutablePath, PathSet({substitutablePath})); + return; + } + } + auto storePath = unpack ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 02ccd714a90..c5f327bd854 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,5 +10,5 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); - + } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index b1782feab06..b77e4907fe4 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "store-api.hh" +#include "archive.hh" #include @@ -58,8 +59,21 @@ std::pair> Input::fetchTree(ref store) if (tree.actualPath == "") tree.actualPath = store->toRealPath(tree.storePath); + if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + { + auto pathOrCa = tree.info.ca + ? StorePathOrDesc {*tree.info.ca} + : StorePathOrDesc {tree.storePath}; + tree.info.narHash = store->queryPathInfo(pathOrCa)->narHash; + } + + + if (!tree.info.narHash) { + HashSink hashSink(htSHA256); + store->narFromPath(tree.storePath, hashSink); + tree.info.narHash = hashSink.finish().first; + } if (input->narHash) assert(input->narHash == tree.info.narHash); @@ -71,4 +85,30 @@ std::pair> Input::fetchTree(ref store) return {std::move(tree), input}; } +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name) +{ + auto ca = StorePathDescriptor { + .name = std::string { name }, + .info = FixedOutputInfo { + ingestionMethod, + hash, + {} + }, + }; + auto substitutablePath = store->makeFixedOutputPathFromCA(ca); + + try { + store->ensurePath(ca); + + debug("using substituted path '%s'", store->printStorePath(substitutablePath)); + + return substitutablePath; + } catch (Error & e) { + debug("substitution of path '%s' failed: %s", store->printStorePath(substitutablePath), e.what()); + } + + return std::nullopt; +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 59a58ae6781..ec73e1f116d 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -6,6 +6,7 @@ #include "tree-info.hh" #include "attrs.hh" #include "url.hh" +#include "content-address.hh" #include @@ -100,4 +101,7 @@ Tree downloadTarball( const std::string & name, bool immutable); +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name); + } diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh index 9d187209742..57a2c5d6b9b 100644 --- a/src/libfetchers/tree-info.hh +++ b/src/libfetchers/tree-info.hh @@ -2,6 +2,7 @@ #include "path.hh" #include "hash.hh" +#include "content-address.hh" #include @@ -12,6 +13,9 @@ namespace nix::fetchers { struct TreeInfo { std::optional narHash; + + std::optional ca; + std::optional revCount; std::optional lastModified; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index cdff2e21a53..d9011737d8e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -106,7 +106,7 @@ std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) void BinaryCacheStore::writeNarInfo(ref narInfo) { - auto narInfoFile = narInfoFileFor(narInfo->path); + auto narInfoFile = narInfoFileFor(bakeCaIfNeeded(narInfo->path)); upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); @@ -296,15 +296,15 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource stats.narInfoWrite++; } -bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) +bool BinaryCacheStore::isValidPathUncached(StorePathOrDesc storePath) { // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even // though they shouldn't. Not easily fixed. - return fileExists(narInfoFileFor(storePath)); + return fileExists(narInfoFileFor(bakeCaIfNeeded(storePath))); } -void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) +void BinaryCacheStore::narFromPath(StorePathOrDesc storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast(); @@ -330,16 +330,17 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) stats.narReadBytes += narSize; } -void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, +void BinaryCacheStore::queryPathInfoUncached(StorePathOrDesc storePath, Callback> callback) noexcept { auto uri = getUri(); - auto storePathS = printStorePath(storePath); + auto actualStorePath = bakeCaIfNeeded(storePath); + auto storePathS = printStorePath(actualStorePath); auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); PushActivity pact(act->id); - auto narInfoFile = narInfoFileFor(storePath); + auto narInfoFile = narInfoFileFor(actualStorePath); auto callbackPtr = std::make_shared(std::move(callback)); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 9bcdf5901b8..66c2ed56c95 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -70,9 +70,9 @@ private: public: - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept override; std::optional queryPathFromHashPart(const std::string & hashPart) override @@ -88,13 +88,13 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(StorePathOrDesc path, Sink & sink) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { unsupported("buildDerivation"); } - void ensurePath(const StorePath & path) override + void ensurePath(StorePathOrDesc path) override { unsupported("ensurePath"); } ref getFSAccessor() override; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index fd96a85fb79..0d1a11d6756 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ class Worker GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + GoalPtr makeSubstitutionGoal(StorePathOrDesc storePath, RepairFlag repair = NoRepair); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -1205,8 +1205,11 @@ void DerivationGoal::haveDerivation() through substitutes. If that doesn't work, we'll build them. */ if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & i : invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv))); + for (auto & i : invalidOutputs) { + auto optCA = getDerivationCA(*drv); + auto p = optCA ? StorePathOrDesc { *optCA } : i; + addWaitee(worker.makeSubstitutionGoal(p, buildMode == bmRepair ? Repair : NoRepair)); + } if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); @@ -1951,7 +1954,7 @@ void linkOrCopy(const Path & from, const Path & to) file (e.g. 32000 of ext3), which is quite possible after a 'nix-store --optimise'. FIXME: actually, why don't we just bind-mount in this case? - + It can also fail with EPERM in BeegFS v7 and earlier versions which don't allow hard-links to other directories */ if (errno != EMLINK && errno != EPERM) @@ -2734,10 +2737,10 @@ struct RestrictedStore : public LocalFSStore return paths; } - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept override { - if (goal.isAllowed(path)) { + if (goal.isAllowed(bakeCaIfNeeded(path))) { try { /* Censor impure information. */ auto info = std::make_shared(*next->queryPathInfo(path)); @@ -2774,7 +2777,7 @@ struct RestrictedStore : public LocalFSStore goal.addDependency(info.path); } - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); @@ -2790,15 +2793,17 @@ struct RestrictedStore : public LocalFSStore return path; } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override { + auto path = bakeCaIfNeeded(pathOrDesc); if (!goal.isAllowed(path)) throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); + LocalFSStore::narFromPath(pathOrDesc, sink); } - void ensurePath(const StorePath & path) override + void ensurePath(StorePathOrDesc pathOrDesc) override { + auto path = bakeCaIfNeeded(pathOrDesc); if (!goal.isAllowed(path)) throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); /* Nothing to be done; 'path' must already be valid. */ @@ -3773,7 +3778,7 @@ void DerivationGoal::registerOutputs() actualPath = actualDest; } else - assert(worker.store.parseStorePath(path) == dest); + assert(i.second.path == dest); ca = StorePathDescriptor { .name = std::string { i.second.path.name() }, @@ -3803,8 +3808,8 @@ void DerivationGoal::registerOutputs() HashResult hash = pathSetAndHash.second; if (buildMode == bmCheck) { - if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; - ValidPathInfo info(*worker.store.queryPathInfo(worker.store.parseStorePath(path))); + if (!worker.store.isValidPath(i.second.path)) continue; + ValidPathInfo info(*worker.store.queryPathInfo(i.second.path)); if (hash.first != info.narHash) { worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { @@ -3847,12 +3852,12 @@ void DerivationGoal::registerOutputs() if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(worker.store.parseStorePath(path)); + worker.markContentsGood(i.second.path); } auto info = ca ? ValidPathInfo { worker.store, StorePathDescriptor { *ca } } - : ValidPathInfo { worker.store.parseStorePath(path) }; + : ValidPathInfo { i.second.path }; info.narHash = hash.first; info.narSize = hash.second; info.setReferencesPossiblyToSelf(std::move(references)); @@ -4435,16 +4440,16 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - auto subPath = storePath; if (ca) { - subPath = sub->makeFixedOutputPathFromCA(*ca); + auto subPath = sub->makeFixedOutputPathFromCA(*ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } try { // FIXME: make async - info = sub->queryPathInfo(subPath); + auto p = ca ? StorePathOrDesc { std::cref(*ca) } : std::cref(storePath); + info = sub->queryPathInfo(p); } catch (InvalidPath &) { tryNext(); return; @@ -4562,15 +4567,10 @@ void SubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); - auto subPath = storePath; - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(*ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } + auto p = ca ? StorePathOrDesc { std::cref(*ca) } : std::cref(storePath); copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + p, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -4703,12 +4703,18 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +GoalPtr Worker::makeSubstitutionGoal(StorePathOrDesc path, RepairFlag repair) { - GoalPtr goal = substitutionGoals[path].lock(); // FIXME + auto p = store.bakeCaIfNeeded(path); + GoalPtr goal = substitutionGoals[p].lock(); // FIXME if (!goal) { - goal = std::make_shared(path, *this, repair, ca); - substitutionGoals.insert_or_assign(path, goal); + auto optCA = std::get_if<1>(&path); + goal = std::make_shared( + p, + *this, + repair, + optCA ? std::optional { *optCA } : std::nullopt); + substitutionGoals.insert_or_assign(p, goal); wakeUp(goal); } return goal; @@ -5146,15 +5152,15 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe } -void LocalStore::ensurePath(const StorePath & path) +void LocalStore::ensurePath(StorePathOrDesc path) { /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {{path}}); + // primeCache(*this, {{path}}); Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); + GoalPtr goal = worker.makeSubstitutionGoal(path, NoRepair); Goals goals = {goal}; worker.run(goals); @@ -5163,8 +5169,10 @@ void LocalStore::ensurePath(const StorePath & path) if (goal->ex) { goal->ex->status = worker.exitStatus(); throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + } else { + auto p = this->bakeCaIfNeeded(path); + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(p)); + } } } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index c8557bda7d4..af0780e0703 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -8,6 +8,7 @@ std::string FixedOutputHash::printMethodAlgo() const { return makeFileIngestionPrefix(method) + printHashType(hash.type); } + std::string makeFileIngestionPrefix(const FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: @@ -17,11 +18,9 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { default: throw Error("impossible, caught both cases"); } + abort(); } -// FIXME Put this somewhere? -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { @@ -30,13 +29,26 @@ std::string renderContentAddress(ContentAddress ca) { + th.hash.to_string(Base32, true); }, [](FixedOutputHash fsh) { - return "fixed:" - + makeFileIngestionPrefix(fsh.method) - + fsh.hash.to_string(Base32, true); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } +static HashType parseHashType_(std::string_view & rest) { + auto hashTypeRaw = splitPrefixTo(rest, ':'); + if (!hashTypeRaw) + throw UsageError("hash must be in form \":\", but found: %s", rest); + return parseHashType(*hashTypeRaw); +}; + +static FileIngestionMethod parseFileIngestionMethod_(std::string_view & rest) { + if (splitPrefix(rest, "r:")) + return FileIngestionMethod::Recursive; + return FileIngestionMethod::Flat; +} + ContentAddress parseContentAddress(std::string_view rawCa) { auto rest = rawCa; @@ -44,22 +56,14 @@ ContentAddress parseContentAddress(std::string_view rawCa) { { auto optPrefix = splitPrefixTo(rest, ':'); if (!optPrefix) - throw UsageError("not a content address because it is not in the form \":\": %s", rawCa); + throw UsageError("not a path-info content address because it is not in the form \":\": %s", rawCa); prefix = *optPrefix; } - auto parseHashType_ = [&](){ - auto hashTypeRaw = splitPrefixTo(rest, ':'); - if (!hashTypeRaw) - throw UsageError("content address hash must be in form \":\", but found: %s", rawCa); - HashType hashType = parseHashType(*hashTypeRaw); - return std::move(hashType); - }; - // Switch on prefix if (prefix == "text") { // No parsing of the method, "text" only support flat. - HashType hashType = parseHashType_(); + HashType hashType = parseHashType_(rest); if (hashType != htSHA256) throw Error("text content address hash should use %s, but instead uses %s", printHashType(htSHA256), printHashType(hashType)); @@ -67,17 +71,14 @@ ContentAddress parseContentAddress(std::string_view rawCa) { .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), }; } else if (prefix == "fixed") { - // Parse method - auto method = FileIngestionMethod::Flat; - if (splitPrefix(rest, "r:")) - method = FileIngestionMethod::Recursive; - HashType hashType = parseHashType_(); + auto method = parseFileIngestionMethod_(rest); + HashType hashType = parseHashType_(rest); return FixedOutputHash { .method = method, .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), }; } else - throw UsageError("content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix); + throw UsageError("path-info content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\", \"fixed\", or \"ipfs\"", prefix); }; std::optional parseContentAddressOpt(std::string_view rawCaOpt) { @@ -88,6 +89,120 @@ std::string renderContentAddress(std::optional ca) { return ca ? renderContentAddress(*ca) : ""; } + +// FIXME Deduplicate with store-api.cc path computation +std::string renderStorePathDescriptor(StorePathDescriptor ca) +{ + std::string result = ca.name; + + auto dumpRefs = [&](auto references, bool hasSelfReference) { + result += "refs:"; + result += std::to_string(references.size()); + for (auto & i : references) { + result += ":"; + result += i.to_string(); + } + if (hasSelfReference) result += ":self"; + }; + + std::visit(overloaded { + [&](TextInfo th) { + result += "text:"; + dumpRefs(th.references, false); + result += ":" + renderContentAddress(ContentAddress {TextHash { + .hash = th.hash, + }}); + }, + [&](FixedOutputInfo fsh) { + result += "fixed:"; + dumpRefs(fsh.references.references, fsh.references.hasSelfReference); + result += ":" + renderContentAddress(ContentAddress {FixedOutputHash { + .method = fsh.method, + .hash = fsh.hash + }}); + }, + }, ca.info); + + return result; +} + + +StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa) +{ + auto rest = rawCa; + + std::string_view name; + std::string_view tag; + { + auto optName = splitPrefixTo(rest, ':'); + auto optTag = splitPrefixTo(rest, ':'); + if (!(optTag && optName)) + throw UsageError("not a content address because it is not in the form \"::\": %s", rawCa); + tag = *optTag; + name = *optName; + } + + auto parseRefs = [&]() -> PathReferences { + if (!splitPrefix(rest, "refs,")) + throw Error("Invalid CA \"%s\", \"%s\" should begin with \"refs:\"", rawCa, rest); + PathReferences ret; + size_t numReferences = 0; + { + auto countRaw = splitPrefixTo(rest, ':'); + if (!countRaw) + throw UsageError("Invalid count"); + numReferences = std::stoi(std::string { *countRaw }); + } + for (size_t i = 0; i < numReferences; i++) { + auto s = splitPrefixTo(rest, ':'); + if (!s) + throw UsageError("Missing reference no. %d", i); + ret.references.insert(StorePath(*s)); + } + if (splitPrefix(rest, "self:")) + ret.hasSelfReference = true; + return ret; + }; + + // Dummy value + ContentAddressWithReferences info = TextInfo { Hash(htSHA256), {} }; + + // Switch on tag + if (tag == "text") { + auto refs = parseRefs(); + if (refs.hasSelfReference) + throw UsageError("Text content addresses cannot have self references"); + auto hashType = parseHashType_(rest); + if (hashType != htSHA256) + throw Error("Text content address hash should use %s, but instead uses %s", + printHashType(htSHA256), printHashType(hashType)); + info = TextInfo { + { + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), + }, + refs.references, + }; + } else if (tag == "fixed") { + auto refs = parseRefs(); + auto method = parseFileIngestionMethod_(rest); + auto hashType = parseHashType_(rest); + info = FixedOutputInfo { + { + .method = method, + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), + }, + refs, + }; + } else + throw UsageError("content address tag \"%s\" is unrecognized. Recogonized tages are \"text\", \"fixed\", or \"ipfs\"", tag); + + return StorePathDescriptor { + .name = std::string { name }, + .info = info, + }; +} + + Hash getContentAddressHash(const ContentAddress & ca) { return std::visit(overloaded { @@ -96,7 +211,7 @@ Hash getContentAddressHash(const ContentAddress & ca) }, [](FixedOutputHash fsh) { return fsh.hash; - } + }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 7d15a04fa4e..7614362b7cb 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -11,10 +11,11 @@ namespace nix { */ enum struct FileIngestionMethod : uint8_t { - Flat = false, - Recursive = true + Flat, + Recursive, }; + struct TextHash { Hash hash; }; @@ -66,6 +67,12 @@ struct PathReferences std::set references; bool hasSelfReference = false; + bool operator == (const PathReferences & other) const + { + return references == other.references + && hasSelfReference == other.hasSelfReference; + } + /* Functions to view references + hasSelfReference as one set, mainly for compatibility's sake. */ StorePathSet referencesPossiblyToSelf(const Ref & self) const; @@ -118,18 +125,24 @@ struct FixedOutputInfo : FixedOutputHash { PathReferences references; }; +typedef std::variant< + TextInfo, + FixedOutputInfo +> ContentAddressWithReferences; + struct StorePathDescriptor { std::string name; - std::variant< - TextInfo, - FixedOutputInfo - > info; + ContentAddressWithReferences info; bool operator < (const StorePathDescriptor & other) const { return name < other.name; + // FIXME second field } - }; +std::string renderStorePathDescriptor(StorePathDescriptor ca); + +StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa); + } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4e4e175366b..b8019234bcb 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -375,21 +375,24 @@ static void performOp(TunnelLogger * logger, ref store, } case wopAddToStore: { - std::string s, baseName; + HashType hashAlgo; + std::string baseName; FileIngestionMethod method; { - bool fixed; uint8_t recursive; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; + bool fixed; + uint8_t recursive; + std::string hashAlgoRaw; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; if (recursive > (uint8_t) FileIngestionMethod::Recursive) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = FileIngestionMethod { recursive }; /* Compatibility hack. */ if (!fixed) { - s = "sha256"; + hashAlgoRaw = "sha256"; method = FileIngestionMethod::Recursive; } + hashAlgo = parseHashType(hashAlgoRaw); } - HashType hashAlgo = parseHashType(s); StringSink savedNAR; TeeSource savedNARSource(from, savedNAR); @@ -407,8 +410,11 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); + StringSource dumpSource { + method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s + }; auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s, + dumpSource, baseName, method, hashAlgo); @@ -735,10 +741,11 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 21) source = std::make_unique(from, to); else { - TeeParseSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.saved.s); - source = std::make_unique(saved); + StringSink saved; + TeeSource tee { from, saved }; + ParseSink sink; + parseDump(sink, tee); + source = std::make_unique(std::move(*saved.s)); } logger->startWork(); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4c1bf567309..22ab3823353 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -60,8 +60,10 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeParseSink tee(source); - parseDump(tee, tee.source); + StringSink saved; + TeeSource tee { source, saved }; + ParseSink ether; + parseDump(ether, tee); uint32_t magic = readInt(source); if (magic != exportMagic) @@ -77,15 +79,15 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.saved.s); - info.narSize = tee.saved.s->size(); + info.narHash = hashString(htSHA256, *saved.s); + info.narSize = saved.s->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.saved.s }; + auto source = StringSource { *saved.s }; addToStore(info, source, NoRepair, checkSigs); res.push_back(info.path); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 683fa5196d7..72e2bce2e70 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,13 +2,14 @@ #include "util.hh" #include "archive.hh" #include "args.hh" +#include "sync.hh" #include #include #include #include #include - +#include namespace nix { @@ -129,8 +130,14 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name) void Settings::requireExperimentalFeature(const std::string & name) { - if (!isExperimentalFeatureEnabled(name)) - throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); + if (!isExperimentalFeatureEnabled(name)) { + if (allowExperimentalFeatures) { + static Sync> warned; + if (warned.lock()->insert(name).second) + warn("feature '%s' is experimental", name); + } else + throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); + } } bool Settings::isWSL1() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 36f00033a59..98c97cd07db 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -350,6 +350,10 @@ public: Setting githubAccessToken{this, "", "github-access-token", "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + Setting allowExperimentalFeatures{this, true, "allow-experimental-features", + "Whether the use of experimental features other than those listed in " + "the option 'experimental-features' gives a warning rather than fatal error."}; + Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5b7de70af0c..20fef329938 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -87,9 +87,10 @@ struct LegacySSHStore : public Store return uriScheme + host; } - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept override { + auto path = this->bakeCaIfNeeded(pathOrDesc); try { auto conn(connections->get()); @@ -180,8 +181,9 @@ struct LegacySSHStore : public Store throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override { + auto path = this->bakeCaIfNeeded(pathOrDesc); auto conn(connections->get()); conn->to << cmdDumpStorePath << printStorePath(path); @@ -233,7 +235,7 @@ struct LegacySSHStore : public Store return status; } - void ensurePath(const StorePath & path) override + void ensurePath(StorePathOrDesc desc) override { unsupported("ensurePath"); } void computeFSClosure(const StorePathSet & paths, diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 2f1d9663ad8..0e1f053b0c9 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -77,11 +77,12 @@ ref LocalFSStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } -void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) +void LocalFSStore::narFromPath(const StorePathOrDesc pathOrDesc, Sink & sink) { - if (!isValidPath(path)) - throw Error("path '%s' is not valid", printStorePath(path)); - dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); + auto p = this->bakeCaIfNeeded(pathOrDesc); + if (!isValidPath(pathOrDesc)) + throw Error("path '%s' is not valid", printStorePath(p)); + dumpPath(getRealStoreDir() + std::string(printStorePath(p), storeDir.size()), sink); } const string LocalFSStore::drvsLogDir = "drvs"; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0f34b6a4e6..3a469de52bb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -629,9 +629,10 @@ uint64_t LocalStore::addValidPath(State & state, } -void LocalStore::queryPathInfoUncached(const StorePath & path, +void LocalStore::queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept { + auto path = bakeCaIfNeeded(pathOrDesc); try { auto info = std::make_shared(path); @@ -712,8 +713,9 @@ bool LocalStore::isValidPath_(State & state, const StorePath & path) } -bool LocalStore::isValidPathUncached(const StorePath & path) +bool LocalStore::isValidPathUncached(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); return retrySQLite([&]() { auto state(_state.lock()); return isValidPath_(*state, path); @@ -1054,88 +1056,26 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) -{ - Hash h = hashString(hashAlgo, dump); - - auto dstPath = makeFixedOutputPath(name, FixedOutputInfo { - { - .method = method, - .hash = h, - }, - {}, - }); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - if (method == FileIngestionMethod::Recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else - writeFile(realPath, dump); - - canonicalisePathMetaData(realPath, -1); - - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash = method == FileIngestionMethod::Recursive - ? HashResult { - hashAlgo == htSHA256 ? h : hashString(htSHA256, dump), - dump.size(), - } - : hashPath(htSHA256, realPath); - - optimisePath(realPath); // FIXME: combine with hashPath() - - ValidPathInfo info(dstPath); - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = FixedOutputHash { .method = method, .hash = h }; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink, filter); + else + readFile(srcPath, sink); + }); + return addToStoreFromDump(*source, name, method, hashAlgo, repair); +} - if (method != FileIngestionMethod::Recursive) - return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair); - - /* For computing the NAR hash. */ - auto sha256Sink = std::make_unique(htSHA256); - /* For computing the store path. In recursive SHA-256 mode, this - is the same as the NAR hash, so no need to do it again. */ - std::unique_ptr hashSink = - hashAlgo == htSHA256 - ? nullptr - : std::make_unique(hashAlgo); +StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) +{ + /* For computing the store path. */ + auto hashSink = std::make_unique(hashAlgo); + TeeSource source { source0, *hashSink }; /* Read the source path into memory, but only if it's up to narBufferSize bytes. If it's larger, write it to a temporary @@ -1143,55 +1083,49 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, destination store path is already valid, we just delete the temporary path. Otherwise, we move it to the destination store path. */ - bool inMemory = true; - std::string nar; - - auto source = sinkToSource([&](Sink & sink) { - - LambdaSink sink2([&](const unsigned char * buf, size_t len) { - (*sha256Sink)(buf, len); - if (hashSink) (*hashSink)(buf, len); - - if (inMemory) { - if (nar.size() + len > settings.narBufferSize) { - inMemory = false; - sink << 1; - sink((const unsigned char *) nar.data(), nar.size()); - nar.clear(); - } else { - nar.append((const char *) buf, len); - } - } - - if (!inMemory) sink(buf, len); - }); - - dumpPath(srcPath, sink2, filter); - }); + bool inMemory = false; + + std::string dump; + + /* Fill out buffer, and decide whether we are working strictly in + memory based on whether we break out because the buffer is full + or the original source is empty */ + while (dump.size() < settings.narBufferSize) { + auto oldSize = dump.size(); + constexpr size_t chunkSize = 1024; + auto want = std::min(chunkSize, settings.narBufferSize - oldSize); + dump.resize(oldSize + want); + auto got = 0; + try { + got = source.read((uint8_t *) dump.data() + oldSize, want); + } catch (EndOfFile &) { + inMemory = true; + break; + } + dump.resize(oldSize + got); + } std::unique_ptr delTempDir; Path tempPath; - try { - /* Wait for the source coroutine to give us some dummy - data. This is so that we don't create the temporary - directory if the NAR fits in memory. */ - readInt(*source); + if (!inMemory) { + /* Drain what we pulled so far, and then keep on pulling */ + StringSource dumpSource { dump }; + ChainSource bothSource { dumpSource, source }; auto tempDir = createTempDir(realStoreDir, "add"); delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; - restorePath(tempPath, *source); + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, bothSource); + else + writeFile(tempPath, bothSource); - } catch (EndOfFile &) { - if (!inMemory) throw; - /* The NAR fits in memory, so we didn't do restorePath(). */ + dump.clear(); } - auto sha256 = sha256Sink->finish(); - - Hash hash = hashSink ? hashSink->finish().first : sha256.first; + auto [hash, size] = hashSink->finish(); ValidPathInfo info { *this, @@ -1206,8 +1140,6 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, }, }, }; - info.narHash = sha256.first; - info.narSize = sha256.second; addTempRoot(info.path); @@ -1227,19 +1159,33 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, autoGC(); if (inMemory) { + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ - StringSource source(nar); - restorePath(realPath, source); + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, dumpSource); + else + writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ if (rename(tempPath.c_str(), realPath.c_str())) throw Error("renaming '%s' to '%s'", tempPath, realPath); } + /* For computing the nar hash. In recursive SHA-256 mode, this + is the same as the store hash, so no need to do it again. */ + auto narHash = std::pair { hash, size }; + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { + HashSink narSink { htSHA256 }; + dumpPath(realPath, narSink); + narHash = narSink.finish(); + } + canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath optimisePath(realPath); + info.narHash = narHash.first; + info.narSize = narHash.second; registerValidPath(info); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index aa72f203848..39e633a2570 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -119,14 +119,14 @@ public: std::string getUri() override; - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc, Callback> callback) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; @@ -154,7 +154,7 @@ public: in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == false). */ - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, @@ -167,7 +167,7 @@ public: BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const StorePath & path) override; + void ensurePath(StorePathOrDesc path) override; void addTempRoot(const StorePath & path) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 13823a933c3..7cc04f7f976 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -33,7 +33,8 @@ void Store::computeFSClosure(const StorePathSet & startPaths, state->pending++; } - queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future> fut) { + auto p = parseStorePath(path); + queryPathInfo(p, {[&, pathS(path)](std::future> fut) { // FIXME: calls to isValidPath() should be async try { diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 58235a7fc20..9709562e24b 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -15,6 +15,8 @@ struct NarInfo : ValidPathInfo std::string system; NarInfo() = delete; + NarInfo(const Store & store, StorePathDescriptor && ca) + : ValidPathInfo(store, std::move(ca)) { } NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f7c91b8fbc2..f67c1f811aa 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -297,8 +297,9 @@ ConnectionHandle RemoteStore::getConnection() } -bool RemoteStore::isValidPathUncached(const StorePath & path) +bool RemoteStore::isValidPathUncached(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(getConnection()); conn->to << wopIsValidPath << printStorePath(path); conn.processStderr(); @@ -407,9 +408,10 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const } -void RemoteStore::queryPathInfoUncached(const StorePath & path, +void RemoteStore::queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept { + auto path = bakeCaIfNeeded(pathOrDesc); try { std::shared_ptr info; { @@ -632,8 +634,9 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD } -void RemoteStore::ensurePath(const StorePath & path) +void RemoteStore::ensurePath(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(getConnection()); conn->to << wopEnsurePath << printStorePath(path); conn.processStderr(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index c96ad48e745..dc6fa18ac6c 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -35,14 +35,14 @@ public: /* Implementations of abstract store API methods. */ - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc, Callback> callback) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; @@ -75,7 +75,7 @@ public: BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const StorePath & path) override; + void ensurePath(StorePathOrDesc path) override; void addTempRoot(const StorePath & path) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1b7dff085e6..7b3365cce42 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -226,8 +226,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore fetches the .narinfo file, rather than first checking for its existence via a HEAD request. Since .narinfos are small, doing a GET is unlikely to be slower than HEAD. */ - bool isValidPathUncached(const StorePath & storePath) override + bool isValidPathUncached(StorePathOrDesc storePathOrDesc) override { + auto storePath = bakeCaIfNeeded(storePathOrDesc); try { queryPathInfo(storePath); return true; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index caae6b596c7..3a973d93dd6 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -40,7 +40,7 @@ class SSHStore : public RemoteStore bool sameMachine() override { return false; } - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override; ref getFSAccessor() override; @@ -68,8 +68,9 @@ class SSHStore : public RemoteStore }; }; -void SSHStore::narFromPath(const StorePath & path, Sink & sink) +void SSHStore::narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(connections->get()); conn->to << wopNarFromPath << printStorePath(path); conn->processStderr(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fa261459433..3457678cd93 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "json.hh" #include "derivations.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include @@ -174,6 +175,17 @@ static std::string makeType( return std::move(type); } +StorePath Store::bakeCaIfNeeded(StorePathOrDesc path) const +{ + return std::visit(overloaded { + [this](std::reference_wrapper storePath) { + return StorePath {storePath}; + }, + [this](std::reference_wrapper ca) { + return makeFixedOutputPathFromCA(ca); + }, + }, path); +} StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { @@ -202,10 +214,6 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -// FIXME Put this somewhere? -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; - StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & info) const { // New template @@ -323,9 +331,9 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) return outputPaths; } -bool Store::isValidPath(const StorePath & storePath) +bool Store::isValidPath(StorePathOrDesc storePath) { - std::string hashPart(storePath.hashPart()); + std::string hashPart { bakeCaIfNeeded(storePath).hashPart() }; { auto state_(state.lock()); @@ -359,7 +367,7 @@ bool Store::isValidPath(const StorePath & storePath) /* Default implementation for stores that only implement queryPathInfoUncached(). */ -bool Store::isValidPathUncached(const StorePath & path) +bool Store::isValidPathUncached(StorePathOrDesc path) { try { queryPathInfo(path); @@ -370,7 +378,7 @@ bool Store::isValidPathUncached(const StorePath & path) } -ref Store::queryPathInfo(const StorePath & storePath) +ref Store::queryPathInfo(StorePathOrDesc storePath) { std::promise> promise; @@ -395,11 +403,13 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual) } -void Store::queryPathInfo(const StorePath & storePath, +void Store::queryPathInfo(StorePathOrDesc pathOrCa, Callback> callback) noexcept { std::string hashPart; + auto storePath = bakeCaIfNeeded(pathOrCa); + try { hashPart = storePath.hashPart(); @@ -433,8 +443,8 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); - queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { + queryPathInfoUncached(pathOrCa, + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -447,11 +457,9 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref(info)); @@ -474,13 +482,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -492,7 +500,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -654,18 +662,22 @@ void Store::buildPaths(const std::vector & paths, BuildMod void copyStorePath(ref srcStore, ref dstStore, - const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) + StorePathOrDesc storePath, RepairFlag repair, CheckSigsFlag checkSigs) { auto srcUri = srcStore->getUri(); auto dstUri = dstStore->getUri(); + // FIXME Use CA when we have it in messages below + + auto actualStorePath = srcStore->bakeCaIfNeeded(storePath); + Activity act(*logger, lvlInfo, actCopyPath, srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri) + ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(actualStorePath), dstUri) : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri) - : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri), - {srcStore->printStorePath(storePath), srcUri, dstUri}); + ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(actualStorePath), srcUri) + : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(actualStorePath), srcUri, dstUri), + {srcStore->printStorePath(actualStorePath), srcUri, dstUri}); PushActivity pact(act.id); auto info = srcStore->queryPathInfo(storePath); @@ -673,19 +685,36 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; // recompute store path on the chance dstStore does it differently - if (info->ca && info->references.empty()) { - auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); - if (dstStore->storeDir == srcStore->storeDir) - assert(info->path == info2->path); - info = info2; + if (auto p = std::get_if>(&storePath)) { + auto ca = static_cast(*p); + // { + // ValidPathInfo srcInfoCA { *srcStore, StorePathDescriptor { ca } }; + // assert((PathReferences &)(*info) == (PathReferences &)srcInfoCA); + // } + if (info->references.empty()) { + auto info2 = make_ref(*info); + ValidPathInfo dstInfoCA { *dstStore, StorePathDescriptor { ca } }; + if (dstStore->storeDir == srcStore->storeDir) + assert(info2->path == info2->path); + info2->path = std::move(dstInfoCA.path); + info2->ca = std::move(dstInfoCA.ca); + info = info2; + } } if (!info->narHash) { StringSink sink; - srcStore->narFromPath({storePath}, sink); + srcStore->narFromPath(storePath, sink); auto info2 = make_ref(*info); - info2->narHash = hashString(htSHA256, *sink.s); + + std::unique_ptr hashSink; + if (!info->ca || !info->hasSelfReference) + hashSink = std::make_unique(htSHA256); + else + hashSink = std::make_unique(htSHA256, std::string(info->path.hashPart())); + (*hashSink)((unsigned char *) sink.s->data(), sink.s->size()); + info2->narHash = hashSink->finish().first; + if (!info->narSize) info2->narSize = sink.s->size(); if (info->ultimate) info2->ultimate = false; info = info2; @@ -709,7 +738,7 @@ void copyStorePath(ref srcStore, ref dstStore, }); srcStore->narFromPath(storePath, wrapperSink); }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(actualStorePath), srcStore->getUri()); }); dstStore->addToStore(*info, *source, repair, checkSigs); @@ -911,12 +940,12 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c TextInfo info { th }; assert(!hasSelfReference); info.references = references; - return std::variant { info }; + return ContentAddressWithReferences { info }; }, [&](FixedOutputHash foh) { FixedOutputInfo info { foh }; info.references = static_cast>(*this); - return std::variant { info }; + return ContentAddressWithReferences { info }; }, }, *ca), }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index cdbc2d188fd..6bd8104a050 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -145,6 +145,12 @@ struct BuildResult } }; +/* Useful for many store functions which can take advantage of content + addresses or work with regular store paths */ +typedef std::variant< + std::reference_wrapper, + std::reference_wrapper +> StorePathOrDesc; class Store : public std::enable_shared_from_this, public Config { @@ -256,6 +262,8 @@ public: StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; + StorePath bakeCaIfNeeded(StorePathOrDesc path) const; + /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ @@ -281,11 +289,11 @@ public: const StorePathSet & references) const; /* Check whether a path is valid. */ - bool isValidPath(const StorePath & path); + bool isValidPath(StorePathOrDesc desc); protected: - virtual bool isValidPathUncached(const StorePath & path); + virtual bool isValidPathUncached(StorePathOrDesc desc); public: @@ -307,15 +315,15 @@ public: /* Query information about a valid path. It is permitted to omit the name part of the store path. */ - ref queryPathInfo(const StorePath & path); + ref queryPathInfo(StorePathOrDesc path); /* Asynchronous version of queryPathInfo(). */ - void queryPathInfo(const StorePath & path, + void queryPathInfo(StorePathOrDesc path, Callback> callback) noexcept; protected: - virtual void queryPathInfoUncached(const StorePath & path, + virtual void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept = 0; public: @@ -373,7 +381,7 @@ public: std::optional expectedCAHash = {}); // FIXME: remove? - virtual StorePath addToStoreFromDump(const string & dump, const string & name, + virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); @@ -385,7 +393,7 @@ public: const StorePathSet & references, RepairFlag repair = NoRepair) = 0; /* Write a NAR dump of a store path. */ - virtual void narFromPath(const StorePath & path, Sink & sink) = 0; + virtual void narFromPath(StorePathOrDesc desc, Sink & sink) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If @@ -408,7 +416,7 @@ public: /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the path). */ - virtual void ensurePath(const StorePath & path) = 0; + virtual void ensurePath(StorePathOrDesc desc) = 0; /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ @@ -627,7 +635,8 @@ public: LocalFSStore(const Params & params); - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(StorePathOrDesc path, Sink & sink) override; + ref getFSAccessor() override; /* Register a permanent GC root. */ @@ -648,7 +657,8 @@ public: /* Copy a path from one store to another. */ void copyStorePath(ref srcStore, ref dstStore, - const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + StorePathOrDesc storePath, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); /* Copy store paths from one store to another. The paths may be copied diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 302b1bb18a0..32d98a6104d 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -63,14 +63,6 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; }; -struct TeeParseSink : ParseSink -{ - StringSink saved; - TeeSource source; - - TeeParseSink(Source & source) : source(source, saved) { } -}; - void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f4b3f11bb77..2671007a78f 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -199,9 +199,8 @@ public: template SysError(const Args & ... args) - : Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index c8b71188fe0..ceb53d133c1 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -92,6 +92,10 @@ void Source::operator () (unsigned char * data, size_t len) } } +void Source::operator () (std::basic_string_view & data) +{ + (*this)((unsigned char *) data.data(), data.size()); +} std::string Source::drain() { @@ -322,5 +326,18 @@ void StringSink::operator () (const unsigned char * data, size_t len) s->append((const char *) data, len); } +size_t ChainSource::read(unsigned char * data, size_t len) +{ + if (useSecond) { + return source2.read(data, len); + } else { + try { + return source1.read(data, len); + } catch (EndOfFile &) { + useSecond = true; + return this->read(data, len); + } + } +} } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8386a499124..c2809b5a18d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -54,6 +54,7 @@ struct Source It blocks until all the requested data is available, or throws an error if it is not going to be available. */ void operator () (unsigned char * data, size_t len); + void operator () (std::basic_string_view & data); /* Store up to ‘len’ in the buffer pointed to by ‘data’, and return the number of bytes stored. It blocks until at least @@ -189,11 +190,12 @@ struct TeeSource : Source size_t read(unsigned char * data, size_t len) { size_t n = orig.read(data, len); - sink(data, len); + sink(data, n); return n; } }; + /* A reader that consumes the original Source until 'size'. */ struct SizedSource : Source { @@ -256,6 +258,19 @@ struct LambdaSource : Source } }; +/* Chain two sources together so after the first is exhausted, the second is + used */ +struct ChainSource : Source +{ + Source & source1, & source2; + bool useSecond = false; + ChainSource(Source & s1, Source & s2) + : source1(s1), source2(s2) + { } + + size_t read(unsigned char * data, size_t len) override; +}; + /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 3af485fa0fe..2170e4c9384 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include #include +#include #include #include diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3641daaec8c..d38657843ea 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -597,4 +597,9 @@ constexpr auto enumerate(T && iterable) } +// C++17 std::visit boilerplate +template struct overloaded : Ts... { using Ts::operator()...; }; +template overloaded(Ts...) -> overloaded; + + } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c992b7d740d..592a10a1f7f 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -703,7 +703,8 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) printMissing(globals.state->store, {{globals.state->store->parseStorePath(drv.queryOutPath())}}); if (globals.dryRun) return; - globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); + auto path = globals.state->store->parseStorePath(drv.queryOutPath()); + globals.state->store->ensurePath(path); } debug(format("switching to new user environment")); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 8e7f09e1244..c143755c06f 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -80,7 +80,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* This is only necessary when installing store paths, e.g., `nix-env -i /nix/store/abcd...-foo'. */ state.store->addTempRoot(state.store->parseStorePath(j.second)); - state.store->ensurePath(state.store->parseStorePath(j.second)); + auto path = state.store->parseStorePath(j.second); + state.store->ensurePath(path); references.insert(state.store->parseStorePath(j.second)); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 887c1844518..320e2c82c2e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -194,10 +194,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -210,7 +210,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { { - .method = recursive, + .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, {}, @@ -355,7 +355,8 @@ static void opQuery(Strings opFlags, Strings opArgs) case qDeriver: for (auto & i : opArgs) { - auto info = store->queryPathInfo(store->followLinksToStorePath(i)); + auto path = store->followLinksToStorePath(i); + auto info = store->queryPathInfo(path); cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver"); } break; @@ -880,9 +881,11 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case cmdDumpStorePath: - store->narFromPath(store->parseStorePath(readString(in)), out); + case cmdDumpStorePath: { + auto path = store->parseStorePath(readString(in)); + store->narFromPath(path, out); break; + } case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); diff --git a/src/nix/command.cc b/src/nix/command.cc index 3651a9e9ca7..b4691e8adc8 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -28,7 +28,8 @@ ref StoreCommand::createStore() void StoreCommand::run() { - run(getStore()); + auto store = getStore(); + run(store); } StorePathsCommand::StorePathsCommand(bool recursive) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fdacf604b5c..41cfc32aeec 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -375,7 +375,8 @@ Path NixRepl::getDerivationPath(Value & v) { if (!drvInfo) throw Error("expression does not evaluate to a derivation, so I can't build it"); Path drvPath = drvInfo->queryDrvPath(); - if (drvPath == "" || !state->store->isValidPath(state->store->parseStorePath(drvPath))) + auto path = state->store->parseStorePath(drvPath); + if (drvPath == "" || !state->store->isValidPath(path)) throw Error("expression did not evaluate to a valid derivation"); return drvPath; } diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a880bdae097..c2439cc8aef 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -133,7 +133,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); - if (!store->isValidPath(store->parseStorePath(userEnv))) + auto path = store->parseStorePath(userEnv); + if (!store->isValidPath(path)) throw Error("directory '%s' is not in the Nix store", userEnv); return profileDir; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index fc7a9765c43..397af8295cc 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -73,14 +73,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -178,7 +178,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); From c9f83f25088222642373a086bba18b043004b634 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Jul 2020 22:57:15 +0000 Subject: [PATCH 085/102] Fix issues with store path descriptor round trip --- src/libfetchers/cache.cc | 5 +++-- src/libstore/content-address.cc | 23 ++++++++++------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 527fc0b06a7..9ceee365ac3 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -51,13 +51,14 @@ struct CacheImpl : Cache ref store, const Attrs & inAttrs, const Attrs & infoAttrs, - const StorePathDescriptor & storePath, + const StorePathDescriptor & storePathDesc, bool immutable) override { _state.lock()->add.use() (attrsToJson(inAttrs).dump()) (attrsToJson(infoAttrs).dump()) - (store->printStorePath(store->makeFixedOutputPathFromCA(storePath))) + // FIXME should use JSON for store path descriptor + (renderStorePathDescriptor(storePathDesc)) (immutable) (time(0)).exec(); } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index af0780e0703..bed155b888d 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -15,8 +15,6 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { return ""; case FileIngestionMethod::Recursive: return "r:"; - default: - throw Error("impossible, caught both cases"); } abort(); } @@ -78,7 +76,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), }; } else - throw UsageError("path-info content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\", \"fixed\", or \"ipfs\"", prefix); + throw UsageError("path-info content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix); }; std::optional parseContentAddressOpt(std::string_view rawCaOpt) { @@ -93,7 +91,8 @@ std::string renderContentAddress(std::optional ca) { // FIXME Deduplicate with store-api.cc path computation std::string renderStorePathDescriptor(StorePathDescriptor ca) { - std::string result = ca.name; + std::string result { ca.name }; + result += ":"; auto dumpRefs = [&](auto references, bool hasSelfReference) { result += "refs:"; @@ -103,23 +102,20 @@ std::string renderStorePathDescriptor(StorePathDescriptor ca) result += i.to_string(); } if (hasSelfReference) result += ":self"; + result += ":"; }; std::visit(overloaded { [&](TextInfo th) { result += "text:"; dumpRefs(th.references, false); - result += ":" + renderContentAddress(ContentAddress {TextHash { - .hash = th.hash, - }}); + result += th.hash.to_string(Base32, true); }, [&](FixedOutputInfo fsh) { result += "fixed:"; dumpRefs(fsh.references.references, fsh.references.hasSelfReference); - result += ":" + renderContentAddress(ContentAddress {FixedOutputHash { - .method = fsh.method, - .hash = fsh.hash - }}); + result += makeFileIngestionPrefix(fsh.method); + result += fsh.hash.to_string(Base32, true); }, }, ca.info); @@ -129,6 +125,7 @@ std::string renderStorePathDescriptor(StorePathDescriptor ca) StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa) { + warn("%s", rawCa); auto rest = rawCa; std::string_view name; @@ -143,7 +140,7 @@ StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa) } auto parseRefs = [&]() -> PathReferences { - if (!splitPrefix(rest, "refs,")) + if (!splitPrefix(rest, "refs:")) throw Error("Invalid CA \"%s\", \"%s\" should begin with \"refs:\"", rawCa, rest); PathReferences ret; size_t numReferences = 0; @@ -194,7 +191,7 @@ StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa) refs, }; } else - throw UsageError("content address tag \"%s\" is unrecognized. Recogonized tages are \"text\", \"fixed\", or \"ipfs\"", tag); + throw UsageError("content address tag \"%s\" is unrecognized. Recogonized tages are \"text\" or \"fixed\"", tag); return StorePathDescriptor { .name = std::string { name }, From 8d01e9fb343cb6eff58b5abf97d8de87e87fd999 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 4 Aug 2020 11:28:49 -0400 Subject: [PATCH 086/102] Move back code to minimize diff size --- src/libstore/path-info.hh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a9ee05dcb0d..f8ba019cb25 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -20,6 +20,16 @@ namespace nix { class Store; + +struct SubstitutablePathInfo : PathReferences +{ + std::optional deriver; + uint64_t downloadSize; /* 0 = unknown or inapplicable */ + uint64_t narSize; /* 0 = unknown */ +}; + +typedef std::map SubstitutablePathInfos; + struct ValidPathInfo : PathReferences { StorePath path; @@ -109,13 +119,4 @@ struct ValidPathInfo : PathReferences typedef list ValidPathInfos; - -struct SubstitutablePathInfo : PathReferences -{ - std::optional deriver; - uint64_t downloadSize; /* 0 = unknown or inapplicable */ - uint64_t narSize; /* 0 = unknown */ -}; - -typedef std::map SubstitutablePathInfos; } From 07f91346668774da9a201259fad135d924cf29fe Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 4 Aug 2020 11:36:27 -0400 Subject: [PATCH 087/102] Remove unused imports --- src/libstore/path-info.hh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index f8ba019cb25..9046dfc1a51 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -1,17 +1,9 @@ #pragma once -// TODO many of thes eare not needed. #include "path.hh" #include "hash.hh" #include "content-address.hh" -#include "serialise.hh" -#include "crypto.hh" -#include "lru-cache.hh" -#include "sync.hh" -#include "globals.hh" -#include "config.hh" - -#include + #include #include From caafdfb77a2f1ab85b040575310357fccf5054d1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Aug 2020 14:29:09 +0000 Subject: [PATCH 088/102] Allow tree hash without nar hash for "has all info" method --- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/git.cc | 9 +++++++-- src/libfetchers/github.cc | 2 +- src/libfetchers/mercurial.cc | 3 ++- src/libfetchers/path.cc | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 39918252b8b..32f27187671 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -84,7 +84,7 @@ Attrs Input::toAttrs() const bool Input::hasAllInfo() const { - return getNarHash() && scheme && scheme->hasAllInfo(*this); + return scheme && scheme->hasAllInfo(*this); } bool Input::operator ==(const Input & other) const diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 9e588546a13..92aab23e6ac 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -94,9 +94,14 @@ struct GitInputScheme : InputScheme { bool maybeDirty = !input.getRef(); bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - return + bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + /* FIXME just requiring tree hash is necessary for substitutions to + work for now, but breaks eval purity. Need a better solution before + upstreaming. */ + return (input.getTreeHash() && !submodules) || ( maybeGetIntAttr(input.attrs, "lastModified") - && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); + && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")) + && input.getNarHash()); } /* FIXME no overriding the tree hash / flake registry support for tree diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index f3c6f2a14f6..ed724e950ab 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -106,7 +106,7 @@ struct GitArchiveInputScheme : InputScheme bool hasAllInfo(const Input & input) override { - return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); + return input.getNarHash() && input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); } Input applyOverrides( diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 906d3ac9828..7f9d9245757 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -71,7 +71,8 @@ struct MercurialInputScheme : InputScheme { // FIXME: ugly, need to distinguish between dirty and clean // default trees. - return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); + return input.getNarHash() + && (input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount")); } Input applyOverrides( diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index a153e38be7a..8b2b288cd1d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -67,7 +67,7 @@ struct PathInputScheme : InputScheme bool hasAllInfo(const Input & input) override { - return true; + return (bool) input.getNarHash(); } std::optional getSourcePath(const Input & input) override From 3273d8a9aa700672fc55fd2012793cc5e52adfd3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Aug 2020 14:29:37 +0000 Subject: [PATCH 089/102] Fix misc other git test failures --- src/libexpr/primops/fetchTree.cc | 13 +++++++++---- src/libfetchers/git.cc | 6 ++++++ tests/git.sh | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 18fcc0ef801..629e8d91807 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -28,10 +28,15 @@ void emitTreeAttrs( // FIXME: support arbitrary input attributes. - auto narHash = input.getNarHash(); - assert(narHash); - mkString(*state.allocAttr(v, state.symbols.create("narHash")), - narHash->to_string(SRI, true)); + if (auto narHash = input.getNarHash()) { + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + narHash->to_string(SRI, true)); + } else if (auto treeHash = input.getTreeHash()) { + mkString(*state.allocAttr(v, state.symbols.create("treeHash")), + treeHash->to_string(SRI, true)); + } else + /* Must have either tree hash or NAR hash */ + assert(false); if (auto rev = input.getRev()) { mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 92aab23e6ac..ea7c5150782 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -439,6 +439,12 @@ struct GitInputScheme : InputScheme AutoDelete delTmpDir(tmpDir, true); PathFilter filter = defaultPathFilter; + if (submodules) { + if (input.getTreeHash()) + throw Error("Cannot fetch specific tree hashes if there are submodules"); + warn("Nix's computed git tree hash will be different when submodules are converted to regular directories"); + } + if (submodules) { Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); diff --git a/tests/git.sh b/tests/git.sh index 6d2f5cd4ca7..689a8d0d37e 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -71,7 +71,7 @@ if [[ -n $(type -p git) ]]; then [ $path2 = $path ] # HEAD should be the same path as tree - path3=$(nix eval --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$repo; ref = \"HEAD\"; gitIngestion = true; }).outPath") + path3=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$repo; ref = \"HEAD\"; gitIngestion = true; }).outPath") [ $path3 = $path ] else echo "Git not installed; skipping Git tests" From 8c0fd953f09ab1ae6bd2caee4a1f2f196b17fd22 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Aug 2020 14:41:15 +0000 Subject: [PATCH 090/102] Fix and document some subtle assertions --- src/libfetchers/git.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index ea7c5150782..432f40b7016 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -208,8 +208,9 @@ struct GitInputScheme : InputScheme -> std::pair { assert(input.getRev() || input.getTreeHash()); - assert(!input.getRev() || _input.getRev() == input.getRev()); - assert(!input.getTreeHash() || _input.getTreeHash() == input.getTreeHash()); + /* If was originally set, that original value must be preserved. */ + assert(!_input.getRev() || _input.getRev() == input.getRev()); + assert(!_input.getTreeHash() || _input.getTreeHash() == input.getTreeHash()); if (!shallow) input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); From 535ad25a58db36a1edb1696e91717a368cd61f5d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 7 Aug 2020 14:02:57 +0000 Subject: [PATCH 091/102] Fix perl bindings build --- perl/lib/Nix/Store.xs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index f5f0eea0d18..203b7eb09b9 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -59,7 +59,8 @@ void setVerbosity(int level) int isValidPath(char * path) CODE: try { - RETVAL = store()->isValidPath(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + RETVAL = store()->isValidPath(storePath); } catch (Error & e) { croak("%s", e.what()); } @@ -70,7 +71,8 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) + auto storePath = store()->parseStorePath(path); + for (auto & i : store()->queryPathInfo(storePath)->referencesPossiblyToSelf()) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -80,7 +82,8 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true); + auto storePath = store()->parseStorePath(path); + auto s = store()->queryPathInfo(storePath)->narHash->to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -90,7 +93,8 @@ SV * queryPathHash(char * path) SV * queryDeriver(char * path) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + auto info = store()->queryPathInfo(storePath); if (!info->deriver) XSRETURN_UNDEF; XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); } catch (Error & e) { @@ -101,7 +105,8 @@ SV * queryDeriver(char * path) SV * queryPathInfo(char * path, int base32) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + auto info = store()->queryPathInfo(storePath); if (!info->deriver) XPUSHs(&PL_sv_undef); else @@ -111,7 +116,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * arr = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) arr))); } catch (Error & e) { From 5c9f8c8d23e9c501f1b080f5311b86d7f243dddb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 3 Sep 2020 18:49:47 +0000 Subject: [PATCH 092/102] git fetcher: fix bad copy paste of "shallow" --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 5f1d18b3a67..dad7702e530 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -83,7 +83,7 @@ struct GitInputScheme : InputScheme if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); if (auto treeHash = input.getTreeHash()) url.query.insert_or_assign("treeHash", treeHash->gitRev()); if (maybeGetBoolAttr(input.attrs, "gitIngestion").value_or((bool) input.getTreeHash())) - url.query.insert_or_assign("shallow", "1"); + url.query.insert_or_assign("gitIngestion", "1"); if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) url.query.insert_or_assign("shallow", "1"); From 4b9263f0af6b2667b5fc3553240a482b559f16e6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 3 Sep 2020 21:21:18 +0000 Subject: [PATCH 093/102] Shore up fetching and git tree hashes quite a bit --- src/libexpr/primops/fetchTree.cc | 7 ++- src/libfetchers/git.cc | 87 +++++++++++++++++++++----------- tests/git.sh | 5 +- 3 files changed, 68 insertions(+), 31 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 6ef4eda375d..2f7de4dc640 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -19,7 +19,7 @@ void emitTreeAttrs( { assert(input.isImmutable()); - state.mkAttrs(v, 8); + state.mkAttrs(v, 10); auto storePath = state.store->printStorePath( state.store->makeFixedOutputPathFromCA(tree.storePath)); @@ -48,6 +48,11 @@ void emitTreeAttrs( mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev()); } + if (auto treeHash = input.getTreeHash()) { + mkString(*state.allocAttr(v, state.symbols.create("treeHash")), treeHash->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortTreeHash")), treeHash->gitShortRev()); + } + if (input.getType() == "git") mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index dad7702e530..947cbb12ff2 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -200,7 +200,7 @@ struct GitInputScheme : InputScheme if (input.getRev()) attrs.insert_or_assign("rev", input.getRev()->gitRev()); if (maybeGetBoolAttr(input.attrs, "gitIngestion").value_or((bool) input.getTreeHash())) - attrs.insert_or_assign("gi", true); + attrs.insert_or_assign("gitIngestion", true); return attrs; }; @@ -319,31 +319,51 @@ struct GitInputScheme : InputScheme {"url", actualUrl}, {"ref", *input.getRef()}, }); + if (ingestionMethod == FileIngestionMethod::Git) + mutableAttrs.insert_or_assign("gitIngestion", true); Path repoDir; if (isLocal) { - if (!input.getRev() && !input.getTreeHash()) - input.attrs.insert_or_assign("rev", - Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); + if (!input.getRev() && !input.getTreeHash()) { + auto getHash = [&](std::string rev) { + return Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", rev })), htSHA1).gitRev(); + }; + input.attrs.insert_or_assign("rev", getHash(*input.getRef())); + input.attrs.insert_or_assign("treeHash", getHash(*input.getRef() + ":")); + } repoDir = actualUrl; } else { if (auto res = getCache()->lookup(store, mutableAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); - if (!input.getRev() || input.getRev() == rev2) { - input.attrs.insert_or_assign("rev", rev2.gitRev()); - return makeResult(res->first, std::move(res->second)); + bool found = false; + + if (std::optional revS = maybeGetStrAttr(res->first, "rev")) { + auto rev2 = Hash::parseAny(*revS, htSHA1); + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); + found = true; + } } - auto treeHash2 = Hash::parseNonSRIUnprefixed(getStrAttr(res->first, "treeHash"), htSHA1); - if (!input.getTreeHash() || input.getTreeHash() == treeHash2) { - input.attrs.insert_or_assign("treeHash", rev2.gitRev()); - return makeResult(res->first, std::move(res->second)); + if (std::optional treeHashS = maybeGetStrAttr(res->first, "treeHash")) { + auto treeHash2 = Hash::parseNonSRIUnprefixed(*treeHashS, htSHA1); + if (!input.getTreeHash() || input.getTreeHash() == treeHash2) { + input.attrs.insert_or_assign("treeHash", treeHash2.gitRev()); + found = true; + } } + + bool correctIngestion = + maybeGetBoolAttr(input.attrs, "gitIngestion").value_or(false) + ? ingestionMethod == FileIngestionMethod::Git + : ingestionMethod == FileIngestionMethod::Recursive; + + if (correctIngestion && found) + return makeResult(res->first, std::move(res->second)); } Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); @@ -409,8 +429,12 @@ struct GitInputScheme : InputScheme utimes(localRefFile.c_str(), times); } - if (!input.getRev() && !input.getTreeHash()) - input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + if (!input.getRev() && !input.getTreeHash()) { + auto rev = Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev(); + input.attrs.insert_or_assign("rev", rev); + input.attrs.insert_or_assign("treeHash", + Hash::parseAny(chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", rev + ":" })), htSHA1).gitRev()); + } } if (input.getTreeHash()) { @@ -426,10 +450,10 @@ struct GitInputScheme : InputScheme // FIXME: check whether rev is an ancestor of ref. - if (input.getRev()) - printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); - else if (input.getTreeHash()) - printTalkative("using tree %s of repo '%s'", input.getTreeHash()->gitRev(), actualUrl); + if (auto rev = input.getRev()) + printTalkative("using revision %s of repo '%s'", rev->gitRev(), actualUrl); + if (auto treeHash = input.getTreeHash()) + printTalkative("using tree %s of repo '%s'", treeHash->gitRev(), actualUrl); /* Now that we know the ref, check again whether we have it in the store. */ @@ -480,27 +504,32 @@ struct GitInputScheme : InputScheme auto storePathDesc = store->queryPathInfo(storePath)->fullStorePathDescriptorOpt().value(); // verify treeHash is what we actually obtained in the nix store - if (input.getTreeHash()) { + if (auto treeHash = input.getTreeHash()) { auto path = store->toRealPath(store->printStorePath(storePath)); auto gotHash = dumpGitHash(htSHA1, path); if (gotHash != input.getTreeHash()) throw Error("Git hash mismatch in input '%s' (%s), expected '%s', got '%s'", - input.to_string(), path, input.getTreeHash()->gitRev(), gotHash.gitRev()); + input.to_string(), path, treeHash->gitRev(), gotHash.gitRev()); } Attrs infoAttrs({}); - if (input.getTreeHash()) { - infoAttrs.insert_or_assign("treeHash", input.getTreeHash()->gitRev()); - infoAttrs.insert_or_assign("revCount", 0); - infoAttrs.insert_or_assign("lastModified", 0); - } else { - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); - infoAttrs.insert_or_assign("rev", input.getRev()->gitRev()); + + if (auto rev = input.getRev()) { + infoAttrs.insert_or_assign("rev", rev->gitRev()); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", rev->gitRev() })); infoAttrs.insert_or_assign("lastModified", lastModified); + } else + infoAttrs.insert_or_assign("lastModified", 0); - if (!shallow) + if (auto treeHash = input.getTreeHash()) + infoAttrs.insert_or_assign("treeHash", treeHash->gitRev()); + + if (!shallow) { + if (auto rev = input.getRev()) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", rev->gitRev() }))); + else + infoAttrs.insert_or_assign("revCount", 0); } if (!_input.getRev() && !_input.getTreeHash()) diff --git a/tests/git.sh b/tests/git.sh index 689a8d0d37e..98850d50fd7 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -70,7 +70,10 @@ if [[ -n $(type -p git) ]]; then path2=$(nix eval --raw --expr "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$treeHash\"; }).outPath" --substituters file://$cacheDir --option substitute true) [ $path2 = $path ] - # HEAD should be the same path as tree + # HEAD should be the same path and tree hash as tree + nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = file://$repo; ref = \"HEAD\"; gitIngestion = true; })" + treeHash2=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$repo; ref = \"HEAD\"; gitIngestion = true; }).treeHash") + [ $treeHash = $treeHash2 ] path3=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$repo; ref = \"HEAD\"; gitIngestion = true; }).outPath") [ $path3 = $path ] else From e04cf6b6bba4520c41353cd97545c90927ba6f43 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 9 Sep 2020 03:47:26 +0000 Subject: [PATCH 094/102] Allow `copyPaths` and `queryValidPaths` to work with StorePathDesc --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/legacy-ssh-store.cc | 14 +++- src/libstore/local-store.cc | 6 +- src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 19 +++-- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 117 +++++++++++++++++++---------- src/libstore/store-api.hh | 16 +++- 8 files changed, 120 insertions(+), 58 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index bd4153e0b2c..096a3c1e96b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -108,7 +108,7 @@ std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) void BinaryCacheStore::writeNarInfo(ref narInfo) { - auto narInfoFile = narInfoFileFor(bakeCaIfNeeded(narInfo->path)); + auto narInfoFile = narInfoFileFor(narInfo->path); upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 07cf2af22a6..63a861c631f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -309,19 +309,27 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig out.insert(i); } - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override { auto conn(connections->get()); + StorePathSet paths2; + for (auto & pathOrDesc : paths) + paths2.insert(bakeCaIfNeeded(pathOrDesc)); + conn->to << cmdQueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths2); conn->to.flush(); - return WorkerProto::read(*this, conn->from); + auto res = WorkerProto::read(*this, conn->from); + std::set res2; + for (auto & r : res) + res2.insert(r); + return res2; } void connect() override diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6ba9c4b89cb..302fb24f08b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -754,11 +754,11 @@ bool LocalStore::isValidPathUncached(StorePathOrDesc pathOrDesc) } -StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +std::set LocalStore::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { - StorePathSet res; + std::set res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(borrowStorePathOrDesc(i))) res.insert(i); return res; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 17e388b1529..355e8b2ef7c 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -125,7 +125,7 @@ public: bool isValidPathUncached(StorePathOrDesc path) override; - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f7742d1530c..7d626820f66 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -302,19 +302,28 @@ bool RemoteStore::isValidPathUncached(StorePathOrDesc pathOrDesc) } -StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +std::set RemoteStore::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - StorePathSet res; + std::set res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(borrowStorePathOrDesc(i))) + res.insert(i); return res; } else { + StorePathSet paths2; + for (auto & pathOrDesc : paths) + paths2.insert(bakeCaIfNeeded(pathOrDesc)); + // FIXME make new version to take advantage of desc case conn->to << wopQueryValidPaths; - WorkerProto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths2); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + auto res = WorkerProto::read(*this, conn->from); + std::set res2; + for (auto & r : res) + res2.insert(r); + return res2; } } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 04efceb8bcd..7acff61f74a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -41,7 +41,7 @@ public: bool isValidPathUncached(StorePathOrDesc path) override; - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8a07b17f99c..d5e4c33aa93 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -180,6 +180,11 @@ static std::string makeType( return std::move(type); } +StorePath Store::bakeCaIfNeeded(const OwnedStorePathOrDesc & path) const +{ + return bakeCaIfNeeded(borrowStorePathOrDesc(path)); +} + StorePath Store::bakeCaIfNeeded(StorePathOrDesc path) const { return std::visit(overloaded { @@ -192,6 +197,15 @@ StorePath Store::bakeCaIfNeeded(StorePathOrDesc path) const }, path); } +StorePathOrDesc borrowStorePathOrDesc(const OwnedStorePathOrDesc & storePathOrDesc) { + // Can't use std::visit as it would copy :( + if (auto p = std::get_if(&storePathOrDesc)) + return *p; + if (auto p = std::get_if(&storePathOrDesc)) + return *p; + abort(); +} + StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { @@ -552,22 +566,38 @@ void Store::queryPathInfo(StorePathOrDesc pathOrCa, StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + std::set paths2; + for (auto & p : paths) + paths2.insert(p); + auto res = queryValidPaths(paths2, maybeSubstitute); + StorePathSet res2; + for (auto & r : res) { + auto p = std::get_if(&r); + assert(p); + res2.insert(*p); + } + return res2; +} + +std::set Store::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { struct State { size_t left; - StorePathSet valid; + std::set valid; std::exception_ptr exc; }; - Sync state_(State{paths.size(), StorePathSet()}); + Sync state_(State{paths.size(), {}}); std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const StorePath & path) { + auto doQuery = [&](const OwnedStorePathOrDesc & path) { checkInterrupt(); - queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { + auto path2 = borrowStorePathOrDesc(path); + queryPathInfo(path2, {[path, this, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); @@ -806,20 +836,25 @@ void copyStorePath(ref srcStore, ref dstStore, } -std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, +void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, + RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) +{ + std::set storePaths2; + for (auto & p : storePaths) + storePaths2.insert(p); + return copyPaths(srcStore, dstStore, storePaths2, repair, checkSigs, substitute); +} + +void copyPaths(ref srcStore, ref dstStore, const std::set & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { auto valid = dstStore->queryValidPaths(storePaths, substitute); - StorePathSet missing; + std::set missing; for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); - std::map pathsMap; - for (auto & path : storePaths) - pathsMap.insert_or_assign(path, path); - - if (missing.empty()) return pathsMap; + if (missing.empty()) return; Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -834,57 +869,57 @@ std::map copyPaths(ref srcStore, ref dstStor ThreadPool pool; - processGraph(pool, - StorePathSet(missing.begin(), missing.end()), - - [&](const StorePath & storePath) { - auto info = srcStore->queryPathInfo(storePath); - auto storePathForDst = storePath; - if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); - if (dstStore->storeDir == srcStore->storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + processGraph(pool, + std::set(missing.begin(), missing.end()), + + [&](const OwnedStorePathOrDesc & storePathOrDesc) -> std::set { + auto info = srcStore->queryPathInfo(borrowStorePathOrDesc(storePathOrDesc)); + + /* If we can "upgrade" the node into a descriptor, do that. */ + if (auto storePathP = std::get_if(&storePathOrDesc)) { + if (auto descOpt = info->fullStorePathDescriptorOpt()) { + auto & desc = *descOpt; + debug("found CA description for path '%s'", srcStore->printStorePath(*storePathP)); + return { desc }; + } } - pathsMap.insert_or_assign(storePath, storePathForDst); - if (dstStore->isValidPath(storePath)) { + if (dstStore->isValidPath(borrowStorePathOrDesc(storePathOrDesc))) { nrDone++; showProgress(); - return StorePathSet(); + return {}; } bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); - return info->references; + std::set res; + for (auto & i : info->references) + res.insert(i); + return res; }, - [&](const StorePath & storePath) { + [&](const OwnedStorePathOrDesc & storePathOrDesc) { checkInterrupt(); - auto info = srcStore->queryPathInfo(storePath); + auto storePathOrDescB = borrowStorePathOrDesc(storePathOrDesc); - auto storePathForDst = storePath; - if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); - if (dstStore->storeDir == srcStore->storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); + auto info = srcStore->queryPathInfo(storePathOrDescB); + + auto descOpt = info->fullStorePathDescriptorOpt(); + if (descOpt) + storePathOrDescB = *descOpt; - if (!dstStore->isValidPath(storePathForDst)) { + if (!dstStore->isValidPath(storePathOrDescB)) { MaintainCount mc(nrRunning); showProgress(); try { - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + copyStorePath(srcStore, dstStore, storePathOrDescB, repair, checkSigs); } catch (Error &e) { nrFailed++; if (!settings.keepGoing) throw e; + auto storePath = dstStore->bakeCaIfNeeded(storePathOrDescB); logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); showProgress(); return; @@ -895,7 +930,7 @@ std::map copyPaths(ref srcStore, ref dstStor showProgress(); }); - return pathsMap; + return; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8aab5615469..6e1b368770e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -172,11 +172,14 @@ struct BuildResult /* Useful for many store functions which can take advantage of content addresses or work with regular store paths */ +typedef std::variant OwnedStorePathOrDesc; typedef std::variant< std::reference_wrapper, std::reference_wrapper > StorePathOrDesc; +StorePathOrDesc borrowStorePathOrDesc(const OwnedStorePathOrDesc &); + struct StoreConfig : public Config { using Config::Config; @@ -327,6 +330,7 @@ public: StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; StorePath bakeCaIfNeeded(StorePathOrDesc path) const; + StorePath bakeCaIfNeeded(const OwnedStorePathOrDesc & path) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -363,6 +367,8 @@ public: /* Query which of the given paths is valid. Optionally, try to substitute missing paths. */ + virtual std::set queryValidPaths(const std::set & paths, + SubstituteFlag maybeSubstitute = NoSubstitute); virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); @@ -741,9 +747,13 @@ void copyStorePath(ref srcStore, ref dstStore, in parallel. They are copied in a topologically sorted order (i.e. if A is a reference of B, then A is copied before B), but the set of store paths is not automatically closed; use copyClosure() for - that. Returns a map of what each path was copied to the dstStore - as. */ -std::map copyPaths(ref srcStore, ref dstStore, + that. */ +void copyPaths(ref srcStore, ref dstStore, + const std::set & storePaths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); +void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, From ae417b58b6d568a36647fee6f08f27a2dceae472 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 8 Aug 2020 14:41:58 +0000 Subject: [PATCH 095/102] Handle `FileIngestionMethod::Git` for CA builds --- src/libstore/build.cc | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ed5eea8beed..5ebe429596b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4,6 +4,7 @@ #include "local-store.hh" #include "util.hh" #include "archive.hh" +#include "git.hh" #include "affinity.hh" #include "builtins.hh" #include "builtins/buildenv.hh" @@ -3989,16 +3990,25 @@ void DerivationGoal::registerOutputs() rewriteOutput(); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; + Hash got { outputHash.hashType }; // Dummy value switch (outputHash.method) { - case FileIngestionMethod::Recursive: + case FileIngestionMethod::Recursive: { + HashModuloSink caSink { outputHash.hashType, oldHashPart }; dumpPath(actualPath, caSink); + got = caSink.finish().first; break; - case FileIngestionMethod::Flat: + } + case FileIngestionMethod::Flat: { + HashModuloSink caSink { outputHash.hashType, oldHashPart }; readFile(actualPath, caSink); + got = caSink.finish().first; + break; + } + case FileIngestionMethod::Git: { + got = dumpGitHash(outputHash.hashType, (Path) tmpDir + "/tmp"); break; } - auto got = caSink.finish().first; + } HashModuloSink narSink { htSHA256, oldHashPart }; dumpPath(actualPath, narSink); auto narHashAndSize = narSink.finish(); From fdbaca87a4583ab9debfc7c20955dfb2a31ffe14 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Oct 2020 15:17:55 +0000 Subject: [PATCH 096/102] Fix improperly-merged revert from upstream --- src/libstore/globals.cc | 13 +++---------- src/libstore/globals.hh | 4 ---- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 62ee9c7839d..c5734852dbf 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,7 +2,6 @@ #include "util.hh" #include "archive.hh" #include "args.hh" -#include "sync.hh" #include "abstract-setting-to-json.hh" #include @@ -10,10 +9,10 @@ #include #include #include -#include #include + namespace nix { @@ -134,14 +133,8 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name) void Settings::requireExperimentalFeature(const std::string & name) { - if (!isExperimentalFeatureEnabled(name)) { - if (allowExperimentalFeatures) { - static Sync> warned; - if (warned.lock()->insert(name).second) - warn("feature '%s' is experimental", name); - } else - throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); - } + if (!isExperimentalFeatureEnabled(name)) + throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); } bool Settings::isWSL1() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 2a135c73394..8c63c5b3482 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -908,10 +908,6 @@ public: value. )"}; - Setting allowExperimentalFeatures{this, true, "allow-experimental-features", - "Whether the use of experimental features other than those listed in " - "the option 'experimental-features' gives a warning rather than fatal error."}; - Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; From f1bcb730a94b4a0546864762b04d5fc1bedd4550 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 22:56:59 +0000 Subject: [PATCH 097/102] Fill in missing comparison operators for content addresses (cherry-pick of 2c21cb672043fcf3c3fd19f89618b37693c0dc62) --- src/libstore/content-address.hh | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 0618dcc11b2..e895d844a3d 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -3,6 +3,7 @@ #include #include "hash.hh" #include "path.hh" +#include "comparator.hh" namespace nix { @@ -46,6 +47,8 @@ std::string renderContentAddressMethod(ContentAddressMethod caMethod); struct TextHash { Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; /// Pair of a hash, and how the file system was ingested @@ -53,6 +56,8 @@ struct FixedOutputHash { FileIngestionMethod method; Hash hash; std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; /* @@ -91,17 +96,13 @@ struct PathReferences std::set references; bool hasSelfReference = false; - bool operator == (const PathReferences & other) const - { - return references == other.references - && hasSelfReference == other.hasSelfReference; - } - /* Functions to view references + hasSelfReference as one set, mainly for compatibility's sake. */ StorePathSet referencesPossiblyToSelf(const Ref & self) const; void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(PathReferences, me->references, me->hasSelfReference); }; template @@ -142,11 +143,15 @@ void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set struct TextInfo : TextHash { // References for the paths, self references disallowed StorePathSet references; + + GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); }; struct FixedOutputInfo : FixedOutputHash { // References for the paths PathReferences references; + + GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); }; typedef std::variant< @@ -158,17 +163,7 @@ struct StorePathDescriptor { std::string name; ContentAddressWithReferences info; - bool operator == (const StorePathDescriptor & other) const - { - return name == other.name; - // FIXME second field - } - - bool operator < (const StorePathDescriptor & other) const - { - return name < other.name; - // FIXME second field - } + GENERATE_CMP(StorePathDescriptor, me->name, me->info); }; std::string renderStorePathDescriptor(StorePathDescriptor ca); From 37ab1891b9de3b6f72032bd6d2fd0f2d3d602b51 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 26 Mar 2022 03:39:18 +0000 Subject: [PATCH 098/102] Fix bug in `copyPaths` References to unknown nodes are filtered. --- src/libstore/store-api.cc | 43 +++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 62ab892acac..c8265ef6a7f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -899,9 +899,35 @@ void copyPaths(ref srcStore, ref dstStore, const std::setqueryValidPaths(storePaths, substitute); - std::set missing; - for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(path); + std::set missing; + + /* If we can "upgrade" the node into a descriptor, do that. We will + need to have a node for this. */ + std::map upgraded; + + for (auto & path : storePaths) { + if (valid.count(path)) continue; + + missing.insert(path); + + auto info = srcStore->queryPathInfo(borrowStorePathOrDesc(path)); + + auto storePathP = std::get_if(&path); + if (!storePathP) continue; + auto & storePath = *storePathP; + + auto descOpt = info->fullStorePathDescriptorOpt(); + if (!descOpt) continue; + auto desc = *std::move(descOpt); + + debug("found CA description '%s' for path '%s'", + renderStorePathDescriptor(desc), + srcStore->printStorePath(storePath)); + + auto [it, _] = missing.insert(std::move(desc)); + const auto & descRef = std::get(*it); + upgraded.insert_or_assign(storePath, &descRef); + } Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -924,9 +950,14 @@ void copyPaths(ref srcStore, ref dstStore, const std::set(&storePathOrDesc)) { - if (auto descOpt = info->fullStorePathDescriptorOpt()) { - auto & desc = *descOpt; - debug("found CA description for path '%s'", srcStore->printStorePath(*storePathP)); + auto & storePath = *storePathP; + auto it = upgraded.find(storePath); + if (it != upgraded.end()) { + auto & [_, descP] = *it; + auto & desc = *descP; + debug("Using found CA description '%s' for path '%s'", + renderStorePathDescriptor(desc), + srcStore->printStorePath(storePath)); return { desc }; } } From 4a55c7b74b0de4f039fd135a86cfa44afc2a264f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 26 Mar 2022 05:07:48 +0000 Subject: [PATCH 099/102] Remove pointless copy --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c8265ef6a7f..bf4099d5450 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -943,7 +943,7 @@ void copyPaths(ref srcStore, ref dstStore, const std::set(pool, - std::set(missing.begin(), missing.end()), + missing, [&](const OwnedStorePathOrDesc & storePathOrDesc) -> std::set { auto info = srcStore->queryPathInfo(borrowStorePathOrDesc(storePathOrDesc)); From 6601ac86421fb05377f113aa52c55da7b9cbe658 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 14 Jan 2023 16:55:13 -0500 Subject: [PATCH 100/102] Revert "Make `ValidPathInfo` have plain `StorePathSet` references like before" This reverts commit b3d91239ae9f21a60057b278ceeff663fb786246. --- perl/lib/Nix/Store.xs | 4 +- src/libexpr/primops.cc | 5 +- src/libstore/binary-cache-store.cc | 8 +-- src/libstore/build/local-derivation-goal.cc | 11 ++-- src/libstore/build/substitution-goal.cc | 10 ++- src/libstore/content-address.cc | 10 --- src/libstore/content-address.hh | 11 +--- src/libstore/daemon.cc | 8 +-- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 6 +- src/libstore/local-store.cc | 10 +-- src/libstore/make-content-addressed.cc | 5 +- src/libstore/misc.cc | 11 ++-- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/nar-info.cc | 2 +- src/libstore/path-info.cc | 48 +++++++++------ src/libstore/path-info.hh | 11 +++- src/libstore/remote-store.cc | 13 ++-- src/libstore/store-api.cc | 14 ++--- src/libutil/reference-set.hh | 68 +++++++++++++++++++++ src/nix-store/dotgraph.cc | 2 +- src/nix-store/graphml.cc | 2 +- src/nix-store/nix-store.cc | 8 +-- src/nix/why-depends.cc | 2 +- 24 files changed, 166 insertions(+), 109 deletions(-) create mode 100644 src/libutil/reference-set.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index f19fb20bf1c..bdb4fa6556e 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -69,7 +69,7 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) + for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->referencesPossiblyToSelf()) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -110,7 +110,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3b32625b1f7..ae573cf4d75 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1544,7 +1544,8 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V StorePathSet refs; if (state.store->isInStore(path)) { try { - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; + // FIXME: Are self references becoming non-self references OK? + refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->referencesPossiblyToSelf(); } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1979,7 +1980,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->references; + refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ac41add2c73..087b376551a 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,11 +180,11 @@ ref BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. */ - for (auto & ref : info.references) + reads, but typically they'll already be cached. Note that + self-references are always valid. */ + for (auto & ref : info.references.others) try { - if (ref != info.path) - queryPathInfo(ref); + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index bb4f929898e..d96858fc076 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2523,10 +2523,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = std::move(refs.others); - if (refs.self) - newInfo0.references.insert(newInfo0.path); + newInfo0.references = rewriteRefs(); return newInfo0; }, @@ -2777,12 +2774,12 @@ void LocalDerivationGoal::checkOutputs(const std::mapsecond.narSize; - for (auto & ref : i->second.references) + for (auto & ref : i->second.referencesPossiblyToSelf()) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) pathsLeft.push(ref); } } @@ -2822,7 +2819,7 @@ void LocalDerivationGoal::checkOutputs(const std::mapreferences) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + for (auto & i : info->references.others) + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -188,9 +187,8 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + for (auto & i : info->references.others) + assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a98e34cb8a5..3b8a773b74c 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -151,16 +151,6 @@ Hash getContentAddressHash(const ContentAddress & ca) }, ca); } -bool StoreReferences::empty() const -{ - return !self && others.empty(); -} - -size_t StoreReferences::size() const -{ - return (self ? 1 : 0) + others.size(); -} - ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 4a50bbee0f3..f8a4d53701d 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,6 +4,7 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" +#include "reference-set.hh" namespace nix { @@ -94,15 +95,7 @@ Hash getContentAddressHash(const ContentAddress & ca); * References set */ -struct StoreReferences { - StorePathSet others; - bool self = false; - - bool empty() const; - size_t size() const; - - GENERATE_CMP(StoreReferences, me->self, me->others); -}; +typedef References StoreReferences; /* * Full content address diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 12596ba4983..605f871fc20 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -336,7 +336,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->references) + for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -758,7 +758,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references); + worker_proto::write(*store, to, i->second.references.possiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -781,7 +781,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references); + worker_proto::write(*store, to, i.second.references.possiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; @@ -863,7 +863,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, from, Phantom {}); + info.setReferencesPossiblyToSelf(worker_proto::read(*store, from, Phantom {})); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 9875da90906..4adf5157338 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->references); + worker_proto::write(*this, teeSink, info->referencesPossiblyToSelf()); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -80,7 +80,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = references; + info.setReferencesPossiblyToSelf(std::move(references)); info.narSize = saved.s.size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 2c9dd26808a..6a694f03427 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -137,7 +137,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom {}); + info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom {})); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -171,7 +171,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize @@ -200,7 +200,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2d03d2d8b6a..b32953f3ff9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -938,7 +938,8 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + info->insertReferencePossiblyToSelf( + parseStorePath(useQueryReferences.getStr(0))); return info; } @@ -1205,7 +1206,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) + for (auto & j : i.referencesPossiblyToSelf()) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1226,7 +1227,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references; + return i == infos.end() ? StorePathSet() : i->second.references.others; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1524,7 +1525,8 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - info.references = references; + // No self reference allowed with text-hashing + info.references.others = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 09f615439cb..5d7945eb1c7 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -28,9 +28,8 @@ std::map makeContentAddressed( StringMap rewrites; StoreReferences refs; - for (auto & ref : oldInfo->references) { - if (ref == path) - refs.self = true; + refs.self = oldInfo->references.self; + for (auto & ref : oldInfo->references.others) { auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index da96dcebcd7..70e97569a9a 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -40,9 +40,8 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references) - if (ref != path) - res.insert(ref); + for (auto & ref : info->references.others) + res.insert(ref); if (includeOutputs && path.isDerivation()) for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) @@ -224,7 +223,7 @@ void Store::queryMissing(const std::vector & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references) + for (auto & ref : info->second.references.others) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -242,7 +241,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references; + return queryPathInfo(path)->references.others; } catch (InvalidPath &) { return StorePathSet(); } @@ -298,7 +297,7 @@ std::map drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); } OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 3e0689534b6..8d4a21daf81 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -248,7 +248,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath(r)); + narInfo->insertReferencePossiblyToSelf(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 071d8355e5c..f54e8f1fc3f 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -63,7 +63,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) throw corrupt(); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 93f91e702fd..2972c0bbeef 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -12,7 +12,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); } @@ -30,25 +30,16 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c .name = std::string { path.name() }, .info = std::visit(overloaded { [&](const TextHash & th) -> ContentAddressWithReferences { - assert(references.count(path) == 0); + assert(!references.self); return TextInfo { th, - .references = references, + .references = references.others, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } return FixedOutputInfo { foh, - .references = { - .others = std::move(refs), - .self = hasSelfReference, - }, + .references = references, }; }, }, *ca), @@ -94,7 +85,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : references) + for (auto & r : referencesPossiblyToSelf()) refs.push_back(std::string(r.to_string())); return refs; } @@ -109,19 +100,36 @@ ValidPathInfo::ValidPathInfo( { std::visit(overloaded { [this](TextInfo && ti) { - this->references = std::move(ti.references); + this->references = { + .others = std::move(ti.references), + .self = false, + }; this->ca = std::move((TextHash &&) ti); }, [this](FixedOutputInfo && foi) { - this->references = std::move(foi.references.others); - if (foi.references.self) - this->references.insert(path); + this->references = std::move(foi.references); this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); } +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return references.possiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return references.insertPossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return references.setPossiblyToSelf(path, std::move(refs)); +} + + ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); @@ -133,7 +141,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = worker_proto::read(store, source, Phantom {}); + info.setReferencesPossiblyToSelf(worker_proto::read(store, source, Phantom {})); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -154,7 +162,7 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, references); + worker_proto::write(store, sink, referencesPossiblyToSelf()); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 476df79c2af..9254835b77d 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -17,20 +17,19 @@ class Store; struct SubstitutablePathInfo { std::optional deriver; - StorePathSet references; + StoreReferences references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; - struct ValidPathInfo { StorePath path; std::optional deriver; Hash narHash; - StorePathSet references; + StoreReferences references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -82,6 +81,12 @@ struct ValidPathInfo /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; + void insertReferencePossiblyToSelf(StorePath && ref); + void setReferencesPossiblyToSelf(StorePathSet && refs); + static const size_t maxSigs = std::numeric_limits::max(); /* Return the number of signatures on this .narinfo that were diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ff57a77ca28..8ea126c65a5 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references.setPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -421,11 +421,12 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto path = parseStorePath(readString(conn->from)); + SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references.setPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -633,7 +634,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + worker_proto::write(*this, sink, info.referencesPossiblyToSelf()); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -644,7 +645,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); - assert(importedPaths.size() <= 1); + assert(importedPaths.empty() == 0); // doesn't include possible self reference } else { @@ -652,7 +653,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c39e50d1451..9446ad132f5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -200,10 +200,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", StoreReferences { - .others = info.references, - .self = false, - }), + makeType(*this, "text", StoreReferences { info.references }), info.hash, name); } @@ -313,7 +310,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references; + return info.references.others; }, [&](const StorePath & path) { @@ -818,7 +815,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += (format("%1%\n") % info->references.size()).str(); - for (auto & j : info->references) + for (auto & j : info->referencesPossiblyToSelf()) s += printStorePath(j) + "\n"; } @@ -880,7 +877,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1208,7 +1205,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!n) throw Error("number expected"); while ((*n)--) { getline(str, s); - info.references.insert(store.parseStorePath(s)); + info.insertReferencePossiblyToSelf(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional(std::move(info)); @@ -1231,7 +1228,6 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } - Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh new file mode 100644 index 00000000000..ac4a9994e5a --- /dev/null +++ b/src/libutil/reference-set.hh @@ -0,0 +1,68 @@ +#pragma once + +#include "comparator.hh" + +#include + +namespace nix { + +template +struct References +{ + std::set others; + bool self = false; + + bool empty() const; + size_t size() const; + + /* Functions to view references + self as one set, mainly for + compatibility's sake. */ + std::set possiblyToSelf(const Ref & self) const; + void insertPossiblyToSelf(const Ref & self, Ref && ref); + void setPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(References, me->others, me->self); +}; + +template +bool References::empty() const +{ + return !self && others.empty(); +} + +template +size_t References::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +template +std::set References::possiblyToSelf(const Ref & selfRef) const +{ + std::set refs { others }; + if (self) + refs.insert(selfRef); + return refs; +} + +template +void References::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) +{ + if (ref == selfRef) + self = true; + else + others.insert(std::move(ref)); +} + +template +void References::setPossiblyToSelf(const Ref & selfRef, std::set && refs) +{ + if (refs.count(selfRef)) { + self = true; + refs.erase(selfRef); + } + + others = refs; +} + +} diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb31..36d774dca76 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -56,7 +56,7 @@ void printDotGraph(ref store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 425d61e53ed..d2eebca7aff 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->references) { + for (auto & p : info->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b261ecc6d3..5cb5aa53afa 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -263,7 +263,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->references); + auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -344,7 +344,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) + for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) paths.insert(p); } else if (query == qReferrers) { @@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->references); + worker_proto::write(*store, out, info->referencesPossiblyToSelf()); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -964,7 +964,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, in, Phantom {}); + info.setReferencesPossiblyToSelf(worker_proto::read(*store, in, Phantom {})); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 76125e5e43d..33cd1360033 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references, + .refs = store->queryPathInfo(path)->references.others, .dist = path == dependencyPath ? 0 : inf }); From b21dd384235bc9863c89ef249bbc1ca58f4ec3fd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Jan 2023 16:38:54 -0500 Subject: [PATCH 101/102] Revert "Don't add `StorePathDescriptor` for now" This reverts commit 4540e7b940ca56db821fe7c7d7d79fafa488f55e. --- src/libfetchers/tarball.cc | 14 +++-- src/libstore/binary-cache-store.cc | 50 +++++++++------- src/libstore/build/local-derivation-goal.cc | 14 +++-- src/libstore/build/substitution-goal.cc | 7 ++- src/libstore/content-address.hh | 7 +++ src/libstore/local-store.cc | 35 ++++++----- src/libstore/make-content-addressed.cc | 14 +++-- src/libstore/nar-info.hh | 4 +- src/libstore/path-info.cc | 64 +++++++++++---------- src/libstore/path-info.hh | 4 +- src/libstore/store-api.cc | 28 ++++----- src/libstore/store-api.hh | 2 +- src/nix/add-to-store.cc | 14 +++-- src/nix/profile.cc | 20 ++++--- 14 files changed, 151 insertions(+), 126 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 3020466105f..155b86cc4a2 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -72,13 +72,15 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, res.data); ValidPathInfo info { *store, - name, - FixedOutputInfo { - { - .method = FileIngestionMethod::Flat, - .hash = hash, + { + .name = name, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + .references = {}, }, - .references = {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9058bb8b119..ac41add2c73 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -307,15 +307,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - name, - FixedOutputInfo { - { - .method = method, - .hash = nar.first, - }, - .references = { - .others = references, - .self = false, + { + .name = std::string { name }, + .info = FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + .references = { + .others = references, + .self = false, + }, }, }, nar.first, @@ -425,15 +427,17 @@ StorePath BinaryCacheStore::addToStore( return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - name, - FixedOutputInfo { - { - .method = method, - .hash = h, - }, - .references = { - .others = references, - .self = false, + { + .name = std::string { name }, + .info = FixedOutputInfo { + { + .method = method, + .hash = h, + }, + .references = { + .others = references, + .self = false, + }, }, }, nar.first, @@ -461,10 +465,12 @@ StorePath BinaryCacheStore::addTextToStore( return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - std::string { name }, - TextInfo { - { .hash = textHash }, - references, + { + .name = std::string { name }, + .info = TextInfo { + { .hash = textHash }, + references, + }, }, nar.first, }; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 98f8cb06100..bb4f929898e 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2475,13 +2475,15 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto got = caSink.finish().first; ValidPathInfo newInfo0 { worker.store, - outputPathName(drv->name, outputName), - FixedOutputInfo { - { - .method = outputHash.method, - .hash = got, + { + .name = outputPathName(drv->name, outputName), + .info = FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, + }, + .references = rewriteRefs(), }, - .references = rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 87fed495c56..994cb4ac2f6 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -95,9 +95,10 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA( - std::string { storePath.name() }, - caWithoutRefs(*ca)); + subPath = sub->makeFixedOutputPathFromCA({ + .name = std::string { storePath.name() }, + .info = caWithoutRefs(*ca), + }); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index c49ab269f0f..4a50bbee0f3 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -132,4 +132,11 @@ typedef std::variant< ContentAddressWithReferences caWithoutRefs(const ContentAddress &); +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + GENERATE_CMP(StorePathDescriptor, me->name, me->info); +}; + } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e55ccab84a7..2d03d2d8b6a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1136,9 +1136,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // Recompute store path so that we can use a different store root. if (path.second) { - subPath = makeFixedOutputPathFromCA( - path.first.name(), - caWithoutRefs(*path.second)); + subPath = makeFixedOutputPathFromCA({ + .name = std::string { path.first.name() }, + .info = caWithoutRefs(*path.second), + }); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1416,18 +1417,21 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); - ContentAddressWithReferences desc = FixedOutputInfo { - { - .method = method, - .hash = hash, - }, - .references = { - .others = references, - .self = false, + auto desc = StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + .references = { + .others = references, + .self = false, + }, }, }; - auto dstPath = makeFixedOutputPathFromCA(name, desc); + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); @@ -1471,12 +1475,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name optimisePath(realPath, repair); - ValidPathInfo info { - *this, - name, - std::move(desc), - narHash.first - }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 3ee64c77a90..09f615439cb 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -49,13 +49,15 @@ std::map makeContentAddressed( ValidPathInfo info { dstStore, - path.name(), - FixedOutputInfo { - { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, + }, + .references = std::move(refs), }, - .references = std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index a4dccb39710..f1e3aabbda7 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -16,8 +16,8 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) - : ValidPathInfo(store, std::move(name), std::move(ca), narHash) + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5944afd0689..93f91e702fd 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,45 +21,48 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::contentAddressWithReferenences() const +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const { if (! ca) return std::nullopt; - return std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { - assert(references.count(path) == 0); - return TextInfo { - th, - .references = references, - }; - }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return FixedOutputInfo { - foh, - .references = { - .others = std::move(refs), - .self = hasSelfReference, - }, - }; - }, - }, *ca); + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](const TextHash & th) -> ContentAddressWithReferences { + assert(references.count(path) == 0); + return TextInfo { + th, + .references = references, + }; + }, + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return FixedOutputInfo { + foh, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, + }; + }, + }, *ca), + }; } bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = contentAddressWithReferenences(); + auto fullCaOpt = fullStorePathDescriptorOpt(); if (! fullCaOpt) return false; - auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -99,10 +102,9 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - std::string_view name, - ContentAddressWithReferences && ca, + StorePathDescriptor && info, Hash narHash) - : path(store.makeFixedOutputPathFromCA(name, ca)) + : path(store.makeFixedOutputPathFromCA(info)) , narHash(narHash) { std::visit(overloaded { @@ -116,7 +118,7 @@ ValidPathInfo::ValidPathInfo( this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, - }, std::move(ca)); + }, std::move(info.info)); } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 663d9454000..476df79c2af 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -77,7 +77,7 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - std::optional contentAddressWithReferenences() const; + std::optional fullStorePathDescriptorOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -100,7 +100,7 @@ struct ValidPathInfo ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const Store & store, - std::string_view name, ContentAddressWithReferences && ca, Hash narHash); + StorePathDescriptor && ca, Hash narHash); virtual ~ValidPathInfo() { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3c0c2670656..c39e50d1451 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -209,17 +209,17 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { [&](const TextInfo & ti) { - return makeTextPath(name, ti); + return makeTextPath(desc.name, ti); }, [&](const FixedOutputInfo & foi) { - return makeFixedOutputPath(name, foi); + return makeFixedOutputPath(desc.name, foi); } - }, ca); + }, desc.info); } @@ -437,13 +437,15 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, ValidPathInfo info { *this, - name, - FixedOutputInfo { - { - .method = method, - .hash = hash, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + .references = {}, }, - .references = {}, }, narHash, }; @@ -995,8 +997,7 @@ void copyStorePath( if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); info2->path = dstStore.makeFixedOutputPathFromCA( - info->path.name(), - info->contentAddressWithReferenences().value()); + info->fullStorePathDescriptorOpt().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1109,8 +1110,7 @@ std::map copyPaths( auto storePathForDst = storePathForSrc; if (currentPathInfo.ca && currentPathInfo.references.empty()) { storePathForDst = dstStore.makeFixedOutputPathFromCA( - currentPathInfo.path.name(), - currentPathInfo.contentAddressWithReferenences().value()); + currentPathInfo.fullStorePathDescriptorOpt().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2d252db843a..d77aea338aa 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -216,7 +216,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 81dbc09a6f2..0b58818c373 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -43,13 +43,15 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - std::move(*namePart), - FixedOutputInfo { - { - .method = std::move(ingestionMethod), - .hash = std::move(hash), + StorePathDescriptor { + .name = *namePart, + .info = FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + .references = {}, }, - .references = {}, }, narHash, }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 345505532c4..aac8e5c81c4 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -200,15 +200,17 @@ struct ProfileManifest ValidPathInfo info { *store, - "profile", - FixedOutputInfo { - { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, - .references = { - .others = std::move(references), - .self = false, + StorePathDescriptor { + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + .references = { + .others = std::move(references), + .self = false, + }, }, }, narHash, From bfcab8513ba814725a79851c407b3d1aa968bc62 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Sep 2023 20:47:27 -0400 Subject: [PATCH 102/102] Introduce experimental feature We need one anyways, and also it helped fix a test failure. --- src/libfetchers/fetchers.cc | 4 +++- src/libfetchers/git.cc | 27 ++++++++++++++++----------- tests/git.sh | 2 ++ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 76947c20021..cc05ccb25fc 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -264,8 +264,10 @@ std::optional Input::getRev() const std::optional Input::getTreeHash() const { - if (auto s = maybeGetStrAttr(attrs, "treeHash")) + if (auto s = maybeGetStrAttr(attrs, "treeHash")) { + settings.requireExperimentalFeature("git-hashing"); return Hash::parseAny(*s, htSHA1); + } return {}; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 9dab08927ab..566d40e8bf2 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -42,7 +42,7 @@ struct GitInputScheme : InputScheme attrs.emplace("type", "git"); for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref" || name == "treehash" || name == "gitIngestion") + if (name == "rev" || name == "ref" || name == "treeHash" || name == "gitIngestion") attrs.emplace(name, value); else if (name == "shallow") attrs.emplace(name, Explicit { value == "1" }); @@ -335,7 +335,8 @@ struct GitInputScheme : InputScheme return Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", rev })), htSHA1).gitRev(); }; input.attrs.insert_or_assign("rev", getHash(*input.getRef())); - input.attrs.insert_or_assign("treeHash", getHash(*input.getRef() + ":")); + if (settings.isExperimentalFeatureEnabled("git-hashing")) + input.attrs.insert_or_assign("treeHash", getHash(*input.getRef() + ":")); } repoDir = actualUrl; @@ -353,11 +354,13 @@ struct GitInputScheme : InputScheme } } - if (std::optional treeHashS = maybeGetStrAttr(res->first, "treeHash")) { - auto treeHash2 = Hash::parseNonSRIUnprefixed(*treeHashS, htSHA1); - if (!input.getTreeHash() || input.getTreeHash() == treeHash2) { - input.attrs.insert_or_assign("treeHash", treeHash2.gitRev()); - found = true; + if (settings.isExperimentalFeatureEnabled("git-hashing")) { + if (std::optional treeHashS = maybeGetStrAttr(res->first, "treeHash")) { + auto treeHash2 = Hash::parseNonSRIUnprefixed(*treeHashS, htSHA1); + if (!input.getTreeHash() || input.getTreeHash() == treeHash2) { + input.attrs.insert_or_assign("treeHash", treeHash2.gitRev()); + found = true; + } } } @@ -442,8 +445,9 @@ struct GitInputScheme : InputScheme if (!input.getRev() && !input.getTreeHash()) { auto rev = Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev(); input.attrs.insert_or_assign("rev", rev); - input.attrs.insert_or_assign("treeHash", - Hash::parseAny(chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", rev + ":" })), htSHA1).gitRev()); + if (settings.isExperimentalFeatureEnabled("git-hashing")) + input.attrs.insert_or_assign("treeHash", + Hash::parseAny(chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", rev + ":" })), htSHA1).gitRev()); } } @@ -561,8 +565,9 @@ struct GitInputScheme : InputScheme } else infoAttrs.insert_or_assign("lastModified", 0); - if (auto treeHash = input.getTreeHash()) - infoAttrs.insert_or_assign("treeHash", treeHash->gitRev()); + if (settings.isExperimentalFeatureEnabled("git-hashing")) + if (auto treeHash = input.getTreeHash()) + infoAttrs.insert_or_assign("treeHash", treeHash->gitRev()); if (!shallow) { if (auto rev = input.getRev()) diff --git a/tests/git.sh b/tests/git.sh index e8194a85888..07d6a8f4c16 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -3,6 +3,8 @@ source common.sh clearStore clearCache +export NIX_CONFIG+=$'\nextra-experimental-features = git-hashing' + try () { hash=$(nix hash git --base16 --type sha1 $TEST_ROOT/hash-path) if test "$hash" != "$1"; then