From d4aff3fe337636c140106be80b1486c7e5ae1783 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 12:21:25 -0500 Subject: [PATCH 01/21] Add initial IpfsStore currently unimplemented --- src/libstore/ipfs.cc | 61 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 src/libstore/ipfs.cc diff --git a/src/libstore/ipfs.cc b/src/libstore/ipfs.cc new file mode 100644 index 00000000000..3672dca2071 --- /dev/null +++ b/src/libstore/ipfs.cc @@ -0,0 +1,61 @@ +#include "binary-cache-store.hh" + +namespace nix { + +class IpfsStore : public BinaryCacheStore +{ + Path daemonUri; + +public: + + IpfsStore( + const Params & params, const Path & _daemonUri) + : BinaryCacheStore(params) + , daemonUri(_daemonUri) + { + if (daemonUri.back() == '/') + daemonUri.pop_back(); + } + + std::string getUri() override + { + return daemonUri; + } + + void init() override + { + } + + bool fileExists(const std::string & path) override + { + return false; + } + + void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) override + { + } + + void getFile(const std::string & path, Sink & sink) override + { + } + + void getFile(const std::string & path, Callback> callback) noexcept override + { + } + +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (std::string(uri, 0, 12) != "ipfs+http://") + return 0; + auto store = std::make_shared(params, std::string(uri, 5)); + store->init(); + return store; +}); + +} From e5281ce3396c44b756199557482cf6c45f340a3a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:00:45 -0500 Subject: [PATCH 02/21] Add ingestionMethodPrefix to get prefix This simplifies code so that references to r: come from ingestionMethodPrefix. --- src/libexpr/primops.cc | 3 +-- src/libstore/path.hh | 10 ++++++++++ src/libstore/store-api.cc | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fcbb75ceb08..c989a2bc895 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -725,8 +725,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { std::move(outPath), - (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") - + printHashType(h.type), + ingestionMethodPrefix(ingestionMethod) + printHashType(h.type), h.to_string(Base16, false), }); } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 5122e742290..4e670406eb1 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -78,6 +78,16 @@ enum struct FileIngestionMethod : uint8_t { Recursive = true }; +inline std::string ingestionMethodPrefix(FileIngestionMethod method) { + switch (method) { + case FileIngestionMethod::Flat: + return ""; + case FileIngestionMethod::Recursive: + return "r:"; + } + throw; +} + struct StorePathWithOutputs { StorePath path; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 86cb20a26b8..f07b2270e0c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -172,20 +172,20 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - FileIngestionMethod recursive, + FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { + if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + ingestionMethodPrefix(method) + hash.to_string(Base16) + ":"), name); } @@ -787,15 +787,17 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const } else if (hasPrefix(ca, "fixed:")) { - FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6)); + FileIngestionMethod method = FileIngestionMethod::Flat; + if (ca.compare(6, 2, "r:") == 0) + method = FileIngestionMethod::Recursive; + Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length())); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; refs.erase(path); } - if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path) + if (store.makeFixedOutputPath(method, hash, path.name(), refs, hasSelfReference) == path) return true; else warn(); @@ -832,11 +834,9 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) { - return "fixed:" - + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") - + hash.to_string(); + return "fixed:" + ingestionMethodPrefix(method) + hash.to_string(); } From cbad8b45e0800617c18246c00164a7285adbe251 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:04:20 -0500 Subject: [PATCH 03/21] =?UTF-8?q?Rename=20some=20variables=20named=20?= =?UTF-8?q?=E2=80=9Crecursive=E2=80=9D=20to=20=E2=80=9Cmethod=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is much less confusing since recursive is no longer a boolean. --- src/libexpr/primops.cc | 14 +++++++------- src/libstore/binary-cache-store.cc | 6 +++--- src/libstore/binary-cache-store.hh | 2 +- src/libstore/build.cc | 18 +++++++++--------- src/libstore/local-store.cc | 22 +++++++++++----------- src/libstore/store-api.cc | 6 +++--- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c989a2bc895..4c72da44ad5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1041,7 +1041,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod recursive, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1072,12 +1072,12 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first - : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair)); + ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); } else @@ -1107,7 +1107,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value Path path; string name; Value * filterFun = nullptr; - auto recursive = FileIngestionMethod::Recursive; + auto method = FileIngestionMethod::Recursive; Hash expectedHash; for (auto & attr : *args[0]->attrs) { @@ -1123,7 +1123,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value state.forceValue(*attr.value, pos); filterFun = attr.value; } else if (n == "recursive") - recursive = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; + method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; else if (n == "sha256") expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else @@ -1134,7 +1134,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value if (name.empty()) name = baseNameOf(path); - addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); + addPath(state, pos, name, path, filterFun, method, expectedHash, v); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a61af4a001c..f13736c58c0 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -327,7 +327,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { // FIXME: some cut&paste from LocalStore::addToStore(). @@ -336,7 +336,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath small files. */ StringSink sink; Hash h; - if (recursive == FileIngestionMethod::Recursive) { + if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); } else { @@ -345,7 +345,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(recursive, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, h, name)); addToStore(info, sink.s, repair, CheckSigs, nullptr); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 1a1ea636c7c..4ff5609ed88 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -79,7 +79,7 @@ public: std::shared_ptr accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 272c4f1ae10..f5c132a8397 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2734,7 +2734,7 @@ struct RestrictedStore : public LocalFSStore { throw Error("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override { throw Error("addToStore"); } @@ -2747,9 +2747,9 @@ struct RestrictedStore : public LocalFSStore } StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { - auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); goal.addDependency(path); return path; } @@ -3692,10 +3692,10 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - FileIngestionMethod recursive; Hash h; - i.second.parseHashInfo(recursive, h); + FileIngestionMethod outputHashMode; Hash h; + i.second.parseHashInfo(outputHashMode, h); - if (recursive == FileIngestionMethod::Flat) { + if (outputHashMode == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3705,11 +3705,11 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = recursive == FileIngestionMethod::Recursive + Hash h2 = outputHashMode == FileIngestionMethod::Recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); if (h != h2) { @@ -3738,7 +3738,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = makeFixedOutputCA(recursive, h2); + ca = makeFixedOutputCA(outputHashMode, h2); } /* Get rid of all weird permissions. This also checks that diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 746f81beb78..80851b59106 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -557,10 +557,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - FileIngestionMethod recursive; Hash h; - out->second.parseHashInfo(recursive, h); + FileIngestionMethod method; Hash h; + out->second.parseHashInfo(method, h); - check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out"); + check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); } else { @@ -1043,11 +1043,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod recursive, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { Hash h = hashString(hashAlgo, dump); - auto dstPath = makeFixedOutputPath(recursive, h, name); + auto dstPath = makeFixedOutputPath(method, h, name); addTempRoot(dstPath); @@ -1067,7 +1067,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (recursive == FileIngestionMethod::Recursive) { + if (method == FileIngestionMethod::Recursive) { StringSource source(dump); restorePath(realPath, source); } else @@ -1080,7 +1080,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (recursive == FileIngestionMethod::Recursive) { + if (method == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else @@ -1091,7 +1091,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam ValidPathInfo info(dstPath.clone()); info.narHash = hash.first; info.narSize = hash.second; - info.ca = makeFixedOutputCA(recursive, h); + info.ca = makeFixedOutputCA(method, h); registerValidPath(info); } @@ -1103,7 +1103,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); @@ -1111,12 +1111,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (recursive == FileIngestionMethod::Recursive) + if (method == FileIngestionMethod::Recursive) dumpPath(srcPath, sink, filter); else sink.s = make_ref(readFile(srcPath)); - return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); + return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f07b2270e0c..81cdaaaff23 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -204,12 +204,12 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, std::pair Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter) const + const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const { - Hash h = recursive == FileIngestionMethod::Recursive + Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(recursive, h, name), h); + return std::make_pair(makeFixedOutputPath(method, h, name), h); } From 0b98d5546a999ca0201357a611f9ecd590791f6b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:06:48 -0500 Subject: [PATCH 04/21] Handle g: prefix in nix store paths --- src/libstore/derivations.cc | 3 +++ src/libstore/path.hh | 7 +++++-- src/libstore/store-api.cc | 6 ++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c68e7b16b1a..793eae4a223 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -17,6 +17,9 @@ void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & has if (string(algo, 0, 2) == "r:") { recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); + } else if (string(algo, 0, 2) == "g:") { + recursive = FileIngestionMethod::Git; + algo = string(algo, 2); } HashType hashType = parseHashType(algo); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 4e670406eb1..052d9a6b3de 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -74,8 +74,9 @@ const size_t storePathHashLen = 32; // i.e. 160 bits const std::string drvExtension = ".drv"; enum struct FileIngestionMethod : uint8_t { - Flat = false, - Recursive = true + Flat, + Recursive, + Git }; inline std::string ingestionMethodPrefix(FileIngestionMethod method) { @@ -84,6 +85,8 @@ inline std::string ingestionMethodPrefix(FileIngestionMethod method) { return ""; case FileIngestionMethod::Recursive: return "r:"; + case FileIngestionMethod::Git: + return "g:"; } throw; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 81cdaaaff23..b128ed85b31 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -118,8 +118,8 @@ string storePathToHash(const Path & path) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:::", where - = "r:" for recursive (path) hashes, or "" for flat - (file) hashes + = "r:" for recursive (path) hashes, "g:" for git + paths, or "" for flat (file) hashes = "md5", "sha1" or "sha256" = base-16 representation of the path or flat hash of the contents of the path (or expected contents of the @@ -790,6 +790,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const FileIngestionMethod method = FileIngestionMethod::Flat; if (ca.compare(6, 2, "r:") == 0) method = FileIngestionMethod::Recursive; + else if (ca.compare(6, 2, "g:") == 0) + method = FileIngestionMethod::Git; Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length())); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; From 859fbcefc0317b4b6786e0f939b8ea9e146baf85 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:11:45 -0500 Subject: [PATCH 05/21] =?UTF-8?q?Don=E2=80=99t=20use=20FileIngestionMethod?= =?UTF-8?q?=20for=20StorePathsCommand?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a different recursive than used in makeFixedOutputPath. --- src/nix/command.cc | 10 +++++----- src/nix/command.hh | 4 ++-- src/nix/copy.cc | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/nix/command.cc b/src/nix/command.cc index ea0ade88e04..71b02771918 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -31,21 +31,21 @@ void StoreCommand::run() run(getStore()); } -StorePathsCommand::StorePathsCommand(FileIngestionMethod recursive) +StorePathsCommand::StorePathsCommand(bool recursive) : recursive(recursive) { - if (recursive == FileIngestionMethod::Recursive) + if (recursive) addFlag({ .longName = "no-recursive", .description = "apply operation to specified paths only", - .handler = {&this->recursive, FileIngestionMethod::Flat}, + .handler = {&this->recursive, false}, }); else addFlag({ .longName = "recursive", .shortName = 'r', .description = "apply operation to closure of the specified paths", - .handler = {&this->recursive, FileIngestionMethod::Recursive}, + .handler = {&this->recursive, true}, }); mkFlag(0, "all", "apply operation to the entire store", &all); @@ -66,7 +66,7 @@ void StorePathsCommand::run(ref store) for (auto & p : toStorePaths(store, realiseMode, installables)) storePaths.push_back(p.clone()); - if (recursive == FileIngestionMethod::Recursive) { + if (recursive) { StorePathSet closure; store->computeFSClosure(storePathsToSet(storePaths), closure, false, false); storePaths.clear(); diff --git a/src/nix/command.hh b/src/nix/command.hh index 09c621b5b10..959d5f19dbc 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -92,7 +92,7 @@ struct StorePathsCommand : public InstallablesCommand { private: - FileIngestionMethod recursive = FileIngestionMethod::Flat; + bool recursive = false; bool all = false; protected: @@ -101,7 +101,7 @@ protected: public: - StorePathsCommand(FileIngestionMethod recursive = FileIngestionMethod::Flat); + StorePathsCommand(bool recursive = false); using StoreCommand::run; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 60aa3f14bd3..c7c38709d19 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -17,7 +17,7 @@ struct CmdCopy : StorePathsCommand SubstituteFlag substitute = NoSubstitute; CmdCopy() - : StorePathsCommand(FileIngestionMethod::Recursive) + : StorePathsCommand(true) { addFlag({ .longName = "from", From 09a9bf048f5be07c20797a98977dd65776bb58be Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:45:42 -0500 Subject: [PATCH 06/21] Use git: instead of g: for ca prefix this is more precise --- src/libstore/derivations.cc | 2 +- src/libstore/path.hh | 2 +- src/libstore/store-api.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 793eae4a223..bef42312bdf 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -17,7 +17,7 @@ void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & has if (string(algo, 0, 2) == "r:") { recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); - } else if (string(algo, 0, 2) == "g:") { + } else if (string(algo, 0, 4) == "git:") { recursive = FileIngestionMethod::Git; algo = string(algo, 2); } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 052d9a6b3de..3374a47486c 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -86,7 +86,7 @@ inline std::string ingestionMethodPrefix(FileIngestionMethod method) { case FileIngestionMethod::Recursive: return "r:"; case FileIngestionMethod::Git: - return "g:"; + return "git:"; } throw; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b128ed85b31..ed8dbcbeabd 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -118,7 +118,7 @@ string storePathToHash(const Path & path) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:::", where - = "r:" for recursive (path) hashes, "g:" for git + = "r:" for recursive (path) hashes, "git:" for git paths, or "" for flat (file) hashes = "md5", "sha1" or "sha256" = base-16 representation of the path or flat hash of @@ -790,7 +790,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const FileIngestionMethod method = FileIngestionMethod::Flat; if (ca.compare(6, 2, "r:") == 0) method = FileIngestionMethod::Recursive; - else if (ca.compare(6, 2, "g:") == 0) + else if (ca.compare(6, 4, "git:") == 0) method = FileIngestionMethod::Git; Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length())); auto refs = cloneStorePathSet(references); From 44bc71b85bb322ebf1d50cea0b8d063bbec30222 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 14:34:33 -0500 Subject: [PATCH 07/21] Rename recursive to method in more places --- src/libstore/derivations.cc | 8 ++++---- src/libstore/derivations.hh | 2 +- src/nix-store/nix-store.cc | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index bef42312bdf..acab418b63a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,16 +9,16 @@ namespace nix { -void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const +void DerivationOutput::parseHashInfo(FileIngestionMethod & method, Hash & hash) const { - recursive = FileIngestionMethod::Flat; + method = FileIngestionMethod::Flat; string algo = hashAlgo; if (string(algo, 0, 2) == "r:") { - recursive = FileIngestionMethod::Recursive; + method = FileIngestionMethod::Recursive; algo = string(algo, 2); } else if (string(algo, 0, 4) == "git:") { - recursive = FileIngestionMethod::Git; + method = FileIngestionMethod::Git; algo = string(algo, 2); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b1224b93bbb..705049d6fa4 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -22,7 +22,7 @@ struct DerivationOutput , hashAlgo(std::move(hashAlgo)) , hash(std::move(hash)) { } - void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; + void parseHashInfo(FileIngestionMethod & method, Hash & hash) const; }; typedef std::map DerivationOutputs; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3a3060ad82d..68fe6f72735 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.empty()) @@ -187,17 +187,17 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo))); + cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, method, hashAlgo))); } /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.size() != 3) @@ -208,7 +208,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(method, Hash(hash, hashAlgo), name))); } From fb83c10c6a54794855196a8ff571d340fa70984f Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 14:36:06 -0500 Subject: [PATCH 08/21] Throw error when using unsupported git file ingestion --- src/libstore/binary-cache-store.cc | 2 ++ src/libstore/remote-store.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f13736c58c0..38c515f229c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -339,6 +339,8 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); + } else if (method == FileIngestionMethod::Git) { + throw Error("cannot add to binary cache store using the git file ingestion method"); } else { auto s = readFile(srcPath); dumpString(s, sink); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e699..4425ab9f07c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -488,6 +488,8 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + if (method == FileIngestionMethod::Git) throw Error("cannot remotely add to store using the git file ingestion method"); + auto conn(getConnection()); Path srcPath(absPath(_srcPath)); From 168c7d87350e87ade17a1773815537d042abedaf Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 27 May 2020 19:30:53 -0400 Subject: [PATCH 09/21] First draft of git parsing --- src/libstore/local-store.cc | 14 ++++++-- src/libutil/archive.cc | 56 +------------------------------- src/libutil/git.cc | 52 +++++++++++++++++++++++++++++ src/libutil/git.hh | 15 +++++++++ src/libutil/restore-sink.hh | 65 +++++++++++++++++++++++++++++++++++++ src/libutil/serialise.cc | 4 +++ src/libutil/serialise.hh | 1 + 7 files changed, 149 insertions(+), 58 deletions(-) create mode 100644 src/libutil/git.cc create mode 100644 src/libutil/git.hh create mode 100644 src/libutil/restore-sink.hh diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b59106..f7af6ebb091 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1067,11 +1067,19 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (method == FileIngestionMethod::Recursive) { + switch method{ + case FileIngestionMethod::Flat: + writeFile(realPath, dump); + break; + case FileIngestionMethod::Recursive: StringSource source(dump); restorePath(realPath, source); - } else - writeFile(realPath, dump); + break; + case FileIngestionMethod::Git: + StringSource source(dump); + restoreGit(realPath, source); + break; + } canonicalisePathMetaData(realPath, -1); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index db544a212ab..a0aeb5f3965 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -12,6 +12,7 @@ #include #include "archive.hh" +#include "restore-sink.hh" #include "util.hh" #include "config.hh" @@ -294,61 +295,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError(format("creating directory '%1%'") % p); - }; - - void createRegularFile(const Path & path) - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError(format("creating file '%1%'") % p); - } - - void isExecutable() - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(unsigned long long len) - { -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError(format("preallocating file of %1% bytes") % len); - } -#endif - } - - void receiveContents(unsigned char * data, unsigned int len) - { - writeFull(fd.get(), data, len); - } - - void createSymlink(const Path & path, const string & target) - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - void restorePath(const Path & path, Source & source) { diff --git a/src/libutil/git.cc b/src/libutil/git.cc new file mode 100644 index 00000000000..7425eda4fa8 --- /dev/null +++ b/src/libutil/git.cc @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +#include // for strcasecmp + +#include +#include +#include +#include +#include + +#include "util.hh" +#include "config.hh" +#include "restore-sink.hh" + +#include "git.hh" + +namespace nix { + + +// Converts a Path to a ParseSink +void restoreGit(const Path & path, Source & source) { + + RestoreSink sink; + sink.dstPath = path; + parseDump(sink, source); + +} + +void parseGit(ParseSink & sink, Source & source) { + parse(sink, source, ""); +} + +static void parse(ParseSink & sink, Source & source, const Path & path) { + uint8_t buf[4]; + + std::basic_string_view buf_v { + (const uint8_t *) & buf, + std::size(buf) + }; + source(buf_v); + if (buf_v.compare((const uint8_t *)"blob")) { + uint8_t space; + source(& space, 1); + } + else { + } +} + +} diff --git a/src/libutil/git.hh b/src/libutil/git.hh new file mode 100644 index 00000000000..554bf41338f --- /dev/null +++ b/src/libutil/git.hh @@ -0,0 +1,15 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" +#include "archive.hh" + +namespace nix { + +void restoreGit(const Path & path, Source & source); + +void parseGit(ParseSink & sink, Source & source); + +static void parse(ParseSink & sink, Source & source, const Path & path); + +} diff --git a/src/libutil/restore-sink.hh b/src/libutil/restore-sink.hh new file mode 100644 index 00000000000..5fcb8c595ab --- /dev/null +++ b/src/libutil/restore-sink.hh @@ -0,0 +1,65 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" +#include "archive.hh" + +namespace nix { + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError(format("creating directory '%1%'") % p); + }; + + void createRegularFile(const Path & path) + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError(format("creating file '%1%'") % p); + } + + void isExecutable() + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(unsigned long long len) + { +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError(format("preallocating file of %1% bytes") % len); + } +#endif + } + + void receiveContents(unsigned char * data, unsigned int len) + { + writeFull(fd.get(), data, len); + } + + void createSymlink(const Path & path, const string & target) + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + + +} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8201549fd7d..79b6048e6ee 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -89,6 +89,10 @@ void Source::operator () (unsigned char * data, size_t len) } } +void Source::operator () (std::basic_string_view & data) +{ + (*this)((unsigned char *) data.data(), data.size()); +} std::string Source::drain() { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a04118512fc..7bcd476aafc 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -54,6 +54,7 @@ struct Source It blocks until all the requested data is available, or throws an error if it is not going to be available. */ void operator () (unsigned char * data, size_t len); + void operator () (std::basic_string_view & data); /* Store up to ‘len’ in the buffer pointed to by ‘data’, and return the number of bytes stored. It blocks until at least From 147764cfae552ba7533bb54c2850460807e92b87 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 21:04:35 -0500 Subject: [PATCH 10/21] Use switch ingestion method in BinaryCacheStore::addToStore --- src/libstore/binary-cache-store.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 38c515f229c..27e7f1eb79b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -336,15 +336,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath small files. */ StringSink sink; Hash h; - if (method == FileIngestionMethod::Recursive) { + switch (method) { + case FileIngestionMethod::Recursive: dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); - } else if (method == FileIngestionMethod::Git) { - throw Error("cannot add to binary cache store using the git file ingestion method"); - } else { + break; + } + case FileIngestionMethod::Flat: auto s = readFile(srcPath); dumpString(s, sink); h = hashString(hashAlgo, s); + break; + case FileIngestionMethod::Git: + throw Error("cannot add to binary cache store using the git file ingestion method"); + default: throw; } ValidPathInfo info(makeFixedOutputPath(method, h, name)); From dbca599c39edae664ec5891a6d7d8c90e0958382 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:00:45 -0500 Subject: [PATCH 11/21] Add ingestionMethodPrefix to get prefix This simplifies code so that references to r: come from ingestionMethodPrefix. --- src/libexpr/primops.cc | 3 +-- src/libstore/path.hh | 10 ++++++++++ src/libstore/store-api.cc | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d458ab2721f..4c72da44ad5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -725,8 +725,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { std::move(outPath), - (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") - + printHashType(h.type), + ingestionMethodPrefix(ingestionMethod) + printHashType(h.type), h.to_string(Base16, false), }); } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 5122e742290..4e670406eb1 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -78,6 +78,16 @@ enum struct FileIngestionMethod : uint8_t { Recursive = true }; +inline std::string ingestionMethodPrefix(FileIngestionMethod method) { + switch (method) { + case FileIngestionMethod::Flat: + return ""; + case FileIngestionMethod::Recursive: + return "r:"; + } + throw; +} + struct StorePathWithOutputs { StorePath path; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d479b86cb11..81cdaaaff23 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -172,20 +172,20 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - FileIngestionMethod recursive, + FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { + if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + ingestionMethodPrefix(method) + hash.to_string(Base16) + ":"), name); } @@ -787,15 +787,17 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const } else if (hasPrefix(ca, "fixed:")) { - FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6)); + FileIngestionMethod method = FileIngestionMethod::Flat; + if (ca.compare(6, 2, "r:") == 0) + method = FileIngestionMethod::Recursive; + Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length())); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; refs.erase(path); } - if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path) + if (store.makeFixedOutputPath(method, hash, path.name(), refs, hasSelfReference) == path) return true; else warn(); @@ -832,11 +834,9 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) { - return "fixed:" - + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") - + hash.to_string(); + return "fixed:" + ingestionMethodPrefix(method) + hash.to_string(); } From 9b452c965277abe43f53943dd52049bd89232bbf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:06:48 -0500 Subject: [PATCH 12/21] Handle g: prefix in nix store paths --- src/libstore/derivations.cc | 3 +++ src/libstore/path.hh | 7 +++++-- src/libstore/store-api.cc | 6 ++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c68e7b16b1a..793eae4a223 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -17,6 +17,9 @@ void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & has if (string(algo, 0, 2) == "r:") { recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); + } else if (string(algo, 0, 2) == "g:") { + recursive = FileIngestionMethod::Git; + algo = string(algo, 2); } HashType hashType = parseHashType(algo); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 4e670406eb1..052d9a6b3de 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -74,8 +74,9 @@ const size_t storePathHashLen = 32; // i.e. 160 bits const std::string drvExtension = ".drv"; enum struct FileIngestionMethod : uint8_t { - Flat = false, - Recursive = true + Flat, + Recursive, + Git }; inline std::string ingestionMethodPrefix(FileIngestionMethod method) { @@ -84,6 +85,8 @@ inline std::string ingestionMethodPrefix(FileIngestionMethod method) { return ""; case FileIngestionMethod::Recursive: return "r:"; + case FileIngestionMethod::Git: + return "g:"; } throw; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 81cdaaaff23..b128ed85b31 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -118,8 +118,8 @@ string storePathToHash(const Path & path) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:::", where - = "r:" for recursive (path) hashes, or "" for flat - (file) hashes + = "r:" for recursive (path) hashes, "g:" for git + paths, or "" for flat (file) hashes = "md5", "sha1" or "sha256" = base-16 representation of the path or flat hash of the contents of the path (or expected contents of the @@ -790,6 +790,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const FileIngestionMethod method = FileIngestionMethod::Flat; if (ca.compare(6, 2, "r:") == 0) method = FileIngestionMethod::Recursive; + else if (ca.compare(6, 2, "g:") == 0) + method = FileIngestionMethod::Git; Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length())); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; From 16c4945101bd51a27509edef7350b5417d4e7a1b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:45:42 -0500 Subject: [PATCH 13/21] Use git: instead of g: for ca prefix this is more precise --- src/libstore/derivations.cc | 2 +- src/libstore/path.hh | 2 +- src/libstore/store-api.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 793eae4a223..bef42312bdf 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -17,7 +17,7 @@ void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & has if (string(algo, 0, 2) == "r:") { recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); - } else if (string(algo, 0, 2) == "g:") { + } else if (string(algo, 0, 4) == "git:") { recursive = FileIngestionMethod::Git; algo = string(algo, 2); } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 052d9a6b3de..3374a47486c 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -86,7 +86,7 @@ inline std::string ingestionMethodPrefix(FileIngestionMethod method) { case FileIngestionMethod::Recursive: return "r:"; case FileIngestionMethod::Git: - return "g:"; + return "git:"; } throw; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b128ed85b31..ed8dbcbeabd 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -118,7 +118,7 @@ string storePathToHash(const Path & path) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:::", where - = "r:" for recursive (path) hashes, "g:" for git + = "r:" for recursive (path) hashes, "git:" for git paths, or "" for flat (file) hashes = "md5", "sha1" or "sha256" = base-16 representation of the path or flat hash of @@ -790,7 +790,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const FileIngestionMethod method = FileIngestionMethod::Flat; if (ca.compare(6, 2, "r:") == 0) method = FileIngestionMethod::Recursive; - else if (ca.compare(6, 2, "g:") == 0) + else if (ca.compare(6, 4, "git:") == 0) method = FileIngestionMethod::Git; Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length())); auto refs = cloneStorePathSet(references); From 6ba83609472faa95b8c27650378504c16dfa319c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 14:34:33 -0500 Subject: [PATCH 14/21] Rename recursive to method in more places --- src/libstore/derivations.cc | 8 ++++---- src/libstore/derivations.hh | 2 +- src/nix-store/nix-store.cc | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index bef42312bdf..acab418b63a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,16 +9,16 @@ namespace nix { -void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const +void DerivationOutput::parseHashInfo(FileIngestionMethod & method, Hash & hash) const { - recursive = FileIngestionMethod::Flat; + method = FileIngestionMethod::Flat; string algo = hashAlgo; if (string(algo, 0, 2) == "r:") { - recursive = FileIngestionMethod::Recursive; + method = FileIngestionMethod::Recursive; algo = string(algo, 2); } else if (string(algo, 0, 4) == "git:") { - recursive = FileIngestionMethod::Git; + method = FileIngestionMethod::Git; algo = string(algo, 2); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b1224b93bbb..705049d6fa4 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -22,7 +22,7 @@ struct DerivationOutput , hashAlgo(std::move(hashAlgo)) , hash(std::move(hash)) { } - void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; + void parseHashInfo(FileIngestionMethod & method, Hash & hash) const; }; typedef std::map DerivationOutputs; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3a3060ad82d..68fe6f72735 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.empty()) @@ -187,17 +187,17 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo))); + cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, method, hashAlgo))); } /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.size() != 3) @@ -208,7 +208,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(method, Hash(hash, hashAlgo), name))); } From 2e16c283b19c744a47710060fa34f4e8aedc8d7a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 14:36:06 -0500 Subject: [PATCH 15/21] Throw error when using unsupported git file ingestion --- src/libstore/binary-cache-store.cc | 2 ++ src/libstore/remote-store.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f13736c58c0..38c515f229c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -339,6 +339,8 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); + } else if (method == FileIngestionMethod::Git) { + throw Error("cannot add to binary cache store using the git file ingestion method"); } else { auto s = readFile(srcPath); dumpString(s, sink); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e699..4425ab9f07c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -488,6 +488,8 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + if (method == FileIngestionMethod::Git) throw Error("cannot remotely add to store using the git file ingestion method"); + auto conn(getConnection()); Path srcPath(absPath(_srcPath)); From 256bbb651f7b8d63354ba50597aeb0f7c9929e36 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 27 May 2020 19:30:53 -0400 Subject: [PATCH 16/21] First draft of git parsing --- src/libstore/local-store.cc | 14 ++++++-- src/libutil/archive.cc | 56 +------------------------------- src/libutil/git.cc | 52 +++++++++++++++++++++++++++++ src/libutil/git.hh | 15 +++++++++ src/libutil/restore-sink.hh | 65 +++++++++++++++++++++++++++++++++++++ src/libutil/serialise.cc | 4 +++ src/libutil/serialise.hh | 1 + 7 files changed, 149 insertions(+), 58 deletions(-) create mode 100644 src/libutil/git.cc create mode 100644 src/libutil/git.hh create mode 100644 src/libutil/restore-sink.hh diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b59106..f7af6ebb091 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1067,11 +1067,19 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (method == FileIngestionMethod::Recursive) { + switch method{ + case FileIngestionMethod::Flat: + writeFile(realPath, dump); + break; + case FileIngestionMethod::Recursive: StringSource source(dump); restorePath(realPath, source); - } else - writeFile(realPath, dump); + break; + case FileIngestionMethod::Git: + StringSource source(dump); + restoreGit(realPath, source); + break; + } canonicalisePathMetaData(realPath, -1); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index db544a212ab..a0aeb5f3965 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -12,6 +12,7 @@ #include #include "archive.hh" +#include "restore-sink.hh" #include "util.hh" #include "config.hh" @@ -294,61 +295,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError(format("creating directory '%1%'") % p); - }; - - void createRegularFile(const Path & path) - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError(format("creating file '%1%'") % p); - } - - void isExecutable() - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(unsigned long long len) - { -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError(format("preallocating file of %1% bytes") % len); - } -#endif - } - - void receiveContents(unsigned char * data, unsigned int len) - { - writeFull(fd.get(), data, len); - } - - void createSymlink(const Path & path, const string & target) - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - void restorePath(const Path & path, Source & source) { diff --git a/src/libutil/git.cc b/src/libutil/git.cc new file mode 100644 index 00000000000..7425eda4fa8 --- /dev/null +++ b/src/libutil/git.cc @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +#include // for strcasecmp + +#include +#include +#include +#include +#include + +#include "util.hh" +#include "config.hh" +#include "restore-sink.hh" + +#include "git.hh" + +namespace nix { + + +// Converts a Path to a ParseSink +void restoreGit(const Path & path, Source & source) { + + RestoreSink sink; + sink.dstPath = path; + parseDump(sink, source); + +} + +void parseGit(ParseSink & sink, Source & source) { + parse(sink, source, ""); +} + +static void parse(ParseSink & sink, Source & source, const Path & path) { + uint8_t buf[4]; + + std::basic_string_view buf_v { + (const uint8_t *) & buf, + std::size(buf) + }; + source(buf_v); + if (buf_v.compare((const uint8_t *)"blob")) { + uint8_t space; + source(& space, 1); + } + else { + } +} + +} diff --git a/src/libutil/git.hh b/src/libutil/git.hh new file mode 100644 index 00000000000..554bf41338f --- /dev/null +++ b/src/libutil/git.hh @@ -0,0 +1,15 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" +#include "archive.hh" + +namespace nix { + +void restoreGit(const Path & path, Source & source); + +void parseGit(ParseSink & sink, Source & source); + +static void parse(ParseSink & sink, Source & source, const Path & path); + +} diff --git a/src/libutil/restore-sink.hh b/src/libutil/restore-sink.hh new file mode 100644 index 00000000000..5fcb8c595ab --- /dev/null +++ b/src/libutil/restore-sink.hh @@ -0,0 +1,65 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" +#include "archive.hh" + +namespace nix { + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError(format("creating directory '%1%'") % p); + }; + + void createRegularFile(const Path & path) + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError(format("creating file '%1%'") % p); + } + + void isExecutable() + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(unsigned long long len) + { +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError(format("preallocating file of %1% bytes") % len); + } +#endif + } + + void receiveContents(unsigned char * data, unsigned int len) + { + writeFull(fd.get(), data, len); + } + + void createSymlink(const Path & path, const string & target) + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + + +} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8201549fd7d..79b6048e6ee 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -89,6 +89,10 @@ void Source::operator () (unsigned char * data, size_t len) } } +void Source::operator () (std::basic_string_view & data) +{ + (*this)((unsigned char *) data.data(), data.size()); +} std::string Source::drain() { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a04118512fc..7bcd476aafc 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -54,6 +54,7 @@ struct Source It blocks until all the requested data is available, or throws an error if it is not going to be available. */ void operator () (unsigned char * data, size_t len); + void operator () (std::basic_string_view & data); /* Store up to ‘len’ in the buffer pointed to by ‘data’, and return the number of bytes stored. It blocks until at least From 85eeb2acc7083de7f725d54798506125f100a8f5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 21:04:35 -0500 Subject: [PATCH 17/21] Use switch ingestion method in BinaryCacheStore::addToStore --- src/libstore/binary-cache-store.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 38c515f229c..27e7f1eb79b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -336,15 +336,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath small files. */ StringSink sink; Hash h; - if (method == FileIngestionMethod::Recursive) { + switch (method) { + case FileIngestionMethod::Recursive: dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); - } else if (method == FileIngestionMethod::Git) { - throw Error("cannot add to binary cache store using the git file ingestion method"); - } else { + break; + } + case FileIngestionMethod::Flat: auto s = readFile(srcPath); dumpString(s, sink); h = hashString(hashAlgo, s); + break; + case FileIngestionMethod::Git: + throw Error("cannot add to binary cache store using the git file ingestion method"); + default: throw; } ValidPathInfo info(makeFixedOutputPath(method, h, name)); From 39df335302d71e9e6de9dca4b5393b26b240b2b0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 18:55:30 -0500 Subject: [PATCH 18/21] More work on git objects --- src/libstore/binary-cache-store.cc | 5 +- src/libstore/local-store.cc | 9 ++- src/libstore/path.hh | 2 +- src/libutil/archive.cc | 1 - src/libutil/archive.hh | 15 +--- src/libutil/{restore-sink.hh => fs-sink.hh} | 14 +++- src/libutil/git.cc | 76 ++++++++++++++++++++- src/libutil/git.hh | 24 ++++++- src/nix/hash.cc | 40 +++++++---- tests/hash.sh | 15 ++++ 10 files changed, 162 insertions(+), 39 deletions(-) rename src/libutil/{restore-sink.hh => fs-sink.hh} (79%) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 27e7f1eb79b..5ca5bd3bb28 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -341,15 +341,14 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); break; - } - case FileIngestionMethod::Flat: + case FileIngestionMethod::Flat: { auto s = readFile(srcPath); dumpString(s, sink); h = hashString(hashAlgo, s); break; + } case FileIngestionMethod::Git: throw Error("cannot add to binary cache store using the git file ingestion method"); - default: throw; } ValidPathInfo info(makeFixedOutputPath(method, h, name)); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f7af6ebb091..39efa650b25 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,5 +1,6 @@ #include "local-store.hh" #include "globals.hh" +#include "git.hh" #include "archive.hh" #include "pathlocks.hh" #include "worker-protocol.hh" @@ -1067,19 +1068,21 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - switch method{ + switch (method) { case FileIngestionMethod::Flat: writeFile(realPath, dump); break; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::Recursive: { StringSource source(dump); restorePath(realPath, source); break; - case FileIngestionMethod::Git: + } + case FileIngestionMethod::Git: { StringSource source(dump); restoreGit(realPath, source); break; } + } canonicalisePathMetaData(realPath, -1); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 3374a47486c..68fcf50103e 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -76,7 +76,7 @@ const std::string drvExtension = ".drv"; enum struct FileIngestionMethod : uint8_t { Flat, Recursive, - Git + Git, }; inline std::string ingestionMethodPrefix(FileIngestionMethod method) { diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index a0aeb5f3965..3670131c291 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -12,7 +12,6 @@ #include #include "archive.hh" -#include "restore-sink.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 768fe25368d..14bf9740d91 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -2,7 +2,7 @@ #include "types.hh" #include "serialise.hh" - +#include "fs-sink.hh" namespace nix { @@ -50,19 +50,6 @@ void dumpPath(const Path & path, Sink & sink, void dumpString(const std::string & s, Sink & sink); -/* FIXME: fix this API, it sucks. */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void isExecutable() { }; - virtual void preallocateContents(unsigned long long size) { }; - virtual void receiveContents(unsigned char * data, unsigned int len) { }; - - virtual void createSymlink(const Path & path, const string & target) { }; -}; - struct TeeSink : ParseSink { TeeSource source; diff --git a/src/libutil/restore-sink.hh b/src/libutil/fs-sink.hh similarity index 79% rename from src/libutil/restore-sink.hh rename to src/libutil/fs-sink.hh index 5fcb8c595ab..c5599d27471 100644 --- a/src/libutil/restore-sink.hh +++ b/src/libutil/fs-sink.hh @@ -2,10 +2,22 @@ #include "types.hh" #include "serialise.hh" -#include "archive.hh" namespace nix { +/* FIXME: fix this API, it sucks. */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void isExecutable() { }; + virtual void preallocateContents(unsigned long long size) { }; + virtual void receiveContents(unsigned char * data, unsigned int len) { }; + + virtual void createSymlink(const Path & path, const string & target) { }; +}; + struct RestoreSink : ParseSink { Path dstPath; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 7425eda4fa8..32ba36677e4 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -13,19 +13,22 @@ #include "util.hh" #include "config.hh" -#include "restore-sink.hh" +#include "hash.hh" #include "git.hh" +using namespace std::string_literals; + namespace nix { +static void parse(ParseSink & sink, Source & source, const Path & path); // Converts a Path to a ParseSink void restoreGit(const Path & path, Source & source) { RestoreSink sink; sink.dstPath = path; - parseDump(sink, source); + parseGit(sink, source); } @@ -49,4 +52,73 @@ static void parse(ParseSink & sink, Source & source, const Path & path) { } } +// TODO stream file into sink, rather than reading into vector +GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink) +{ + auto s = (format("blob %d\0%s"s) % std::to_string(st.st_size) % readFile(path)).str(); + + vector v; + std::copy(s.begin(), s.end(), std::back_inserter(v)); + sink(v.data(), v.size()); + return st.st_mode & S_IXUSR + ? GitMode::Executable + : GitMode::Regular; +} + +GitMode dumpGitTree(const GitTree & entries, Sink & sink) +{ + std::string s1 = ""; + for (auto & i : entries) { + unsigned int mode; + switch (i.second.first) { + case GitMode::Directory: mode = 40000; break; + case GitMode::Executable: mode = 100755; break; + case GitMode::Regular: mode = 100644; break; + } + s1 += (format("%6d %s\0%s"s) % mode % i.first % i.second.second).str(); + } + + std::string s2 = (format("tree %d\0%s"s) % s1.size() % s1).str(); + + vector v; + std::copy(s2.begin(), s2.end(), std::back_inserter(v)); + sink(v.data(), v.size()); + return GitMode::Directory; +} + +// Returns the perm in addition +std::pair dumpGitHashInternal( + std::function()> makeHashSink, + const Path & path, PathFilter & filter) +{ + auto hashSink = makeHashSink(); + struct stat st; + GitMode perm; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + + if (S_ISREG(st.st_mode)) { + perm = dumpGitBlob(path, st, *hashSink); + } else if (S_ISDIR(st.st_mode)) { + GitTree entries; + for (auto & i : readDirectory(path)) + if (filter(path + "/" + i.name)) { + entries[i.name] = dumpGitHashInternal(makeHashSink, path + "/" + i.name, filter); + } + perm = dumpGitTree(entries, *hashSink); + } else { + throw Error(format("file '%1%' has an unsupported type") % path); + } + + auto hash = hashSink->finish().first; + return std::pair { perm, hash }; +} + +Hash dumpGitHash( + std::function()> makeHashSink, + const Path & path, PathFilter & filter) +{ + return dumpGitHashInternal(makeHashSink, path, filter).second; +} + } diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 554bf41338f..7b11f72be29 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -2,14 +2,34 @@ #include "types.hh" #include "serialise.hh" -#include "archive.hh" +#include "fs-sink.hh" namespace nix { +enum struct GitMode { + Directory, + Executable, + Regular, +}; + void restoreGit(const Path & path, Source & source); void parseGit(ParseSink & sink, Source & source); -static void parse(ParseSink & sink, Source & source, const Path & path); +// Dumps a single file to a sink +GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink); + +typedef std::map> GitTree; + +// Dumps a representation of a git tree to a sink +GitMode dumpGitTree(const GitTree & entries, Sink & sink); + +// Recursively dumps path, hashing as we go +Hash dumpGitHash( + std::function()>, + const Path & path, + PathFilter & filter = defaultPathFilter); +// N.B. There is no way to recursively dump to a sink, as that doesn't make +// sense with the git hash/data model where the information is Merklized. } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 0a7c343d0dd..4f3b3cd426f 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -4,6 +4,7 @@ #include "shared.hh" #include "references.hh" #include "archive.hh" +#include "git.hh" using namespace nix; @@ -41,6 +42,8 @@ struct CmdHash : Command d = "print cryptographic hash of a regular file"; case FileIngestionMethod::Recursive: d = "print cryptographic hash of the NAR serialisation of a path"; + case FileIngestionMethod::Git: + d = "print cryptographic hash of the Git serialisation of a path"; }; return d; } @@ -51,22 +54,34 @@ struct CmdHash : Command { for (auto path : paths) { - std::unique_ptr hashSink; - if (modulus) - hashSink = std::make_unique(ht, *modulus); - else - hashSink = std::make_unique(ht); + auto makeHashSink = [&]() -> std::unique_ptr { + std::unique_ptr t; + if (modulus) + t = std::make_unique(ht, *modulus); + else + t = std::make_unique(ht); + return t; + }; + Hash h; switch (mode) { - case FileIngestionMethod::Flat: + case FileIngestionMethod::Flat: { + auto hashSink = makeHashSink(); readFile(path, *hashSink); + h = hashSink->finish().first; break; - case FileIngestionMethod::Recursive: + } + case FileIngestionMethod::Recursive: { + auto hashSink = makeHashSink(); dumpPath(path, *hashSink); + h = hashSink->finish().first; + break; + } + case FileIngestionMethod::Git: + h = dumpGitHash(makeHashSink, path); break; } - Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); logger->stdout(h.to_string(base, base == SRI)); } @@ -75,6 +90,7 @@ struct CmdHash : Command static RegisterCommand r1("hash-file", [](){ return make_ref(FileIngestionMethod::Flat); }); static RegisterCommand r2("hash-path", [](){ return make_ref(FileIngestionMethod::Recursive); }); +static RegisterCommand r3("hash-git", [](){ return make_ref(FileIngestionMethod::Git); }); struct CmdToBase : Command { @@ -106,10 +122,10 @@ struct CmdToBase : Command } }; -static RegisterCommand r3("to-base16", [](){ return make_ref(Base16); }); -static RegisterCommand r4("to-base32", [](){ return make_ref(Base32); }); -static RegisterCommand r5("to-base64", [](){ return make_ref(Base64); }); -static RegisterCommand r6("to-sri", [](){ return make_ref(SRI); }); +static RegisterCommand r4("to-base16", [](){ return make_ref(Base16); }); +static RegisterCommand r5("to-base32", [](){ return make_ref(Base32); }); +static RegisterCommand r6("to-base64", [](){ return make_ref(Base64); }); +static RegisterCommand r7("to-sri", [](){ return make_ref(SRI); }); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) diff --git a/tests/hash.sh b/tests/hash.sh index 4cfc9790101..33b5bef7595 100644 --- a/tests/hash.sh +++ b/tests/hash.sh @@ -85,3 +85,18 @@ try3() { try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" + +# Git. +try4 () { + hash=$(nix hash-git --base16 --type sha1 $TEST_ROOT/hash-path) + if test "$hash" != "$1"; then + echo "git hash, expected $1, got $hash" + exit 1 + fi +} + +rm -rf $TEST_ROOT/hash-path +mkdir $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path/hello + +try4 "117c62a8c5e01758bd284126a6af69deab9dbbe2" From ffc1024139d6ba6aa50cff7cb382e64fec7cad4b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 11:42:07 -0500 Subject: [PATCH 19/21] Add fcntl.h to fs-sink.hh --- src/libutil/fs-sink.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index c5599d27471..d3d1e43b79f 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -1,5 +1,7 @@ #pragma once +#include + #include "types.hh" #include "serialise.hh" From 35001b6403c95a49545f5da283f51994b8f4cf34 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 27 May 2020 12:29:59 -0400 Subject: [PATCH 20/21] Remove `aarch64-linux` from supportedSystems until we have a builder --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 2a320e1c3bc..fa8e8eb8499 100644 --- a/release.nix +++ b/release.nix @@ -1,7 +1,7 @@ { nix ? builtins.fetchGit ./. , nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz , officialRelease ? false -, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] +, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" ] }: let From 9d820063371574fa4fd3f2d07a5362d1c2acec34 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 27 May 2020 14:17:13 -0400 Subject: [PATCH 21/21] Remove x86-linux from supportedSystems for now, too --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index fa8e8eb8499..5acccf276b8 100644 --- a/release.nix +++ b/release.nix @@ -1,7 +1,7 @@ { nix ? builtins.fetchGit ./. , nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz , officialRelease ? false -, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" ] +, systems ? [ "x86_64-linux" "x86_64-darwin" ] }: let