From 130267de37e799f77f9f7d1edc8063126af9e640 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 17:22:40 -0500 Subject: [PATCH 01/35] Handle git ingestion over daemon --- src/libstore/daemon.cc | 19 ++++++++++++++++--- src/libstore/local-store.cc | 18 +++++++++++++++--- src/libstore/remote-store.cc | 2 +- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f1afdff699e..aa2cbe6d487 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -8,6 +8,7 @@ #include "archive.hh" #include "derivations.hh" #include "args.hh" +#include "git.hh" namespace nix::daemon { @@ -358,7 +359,8 @@ static void performOp(TunnelLogger * logger, ref store, std::string s, baseName; FileIngestionMethod method; { - bool fixed, recursive; + bool fixed; + unsigned char recursive; from >> baseName >> fixed /* obsolete */ >> recursive >> s; method = FileIngestionMethod { recursive }; /* Compatibility hack. */ @@ -372,14 +374,25 @@ static void performOp(TunnelLogger * logger, ref store, TeeSource savedNAR(from); RetrieveRegularNARSink savedRegular; - if (method == FileIngestionMethod::Recursive) { + switch (method) { + case FileIngestionMethod::Recursive: { /* Get the entire NAR dump from the client and save it to a string so that we can pass it to addToStoreFromDump(). */ ParseSink sink; /* null sink; just parse the NAR */ parseDump(sink, savedNAR); - } else + break; + } + case FileIngestionMethod::Git: { + ParseSink sink; + parseGit(sink, savedNAR); + break; + } + case FileIngestionMethod::Flat: { parseDump(savedRegular, from); + break; + } + } logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 39efa650b25..e214c334b73 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1017,7 +1017,10 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, return n; }); - restorePath(realPath, wrapperSource); + if (hasPrefix(info.ca, "fixed:git:")) + restoreGit(realPath, wrapperSource); + else + restorePath(realPath, wrapperSource); auto hashResult = hashSink->finish(); @@ -1122,10 +1125,19 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (method == FileIngestionMethod::Recursive) + switch (method) { + case FileIngestionMethod::Recursive: dumpPath(srcPath, sink, filter); - else + break; + case FileIngestionMethod::Git:{ + dumpGit(srcPath, sink, filter); + break; + } + case FileIngestionMethod::Flat: { sink.s = make_ref(readFile(srcPath)); + break; + } + } return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4425ab9f07c..842c2e17666 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -498,7 +498,7 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, << wopAddToStore << name << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (method == FileIngestionMethod::Recursive ? 1 : 0) + << (uint8_t) method << printHashType(hashAlgo); try { From df747447bf3cfd965fec37f53cac2c04fc8b5d4b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 17:23:05 -0500 Subject: [PATCH 02/35] Parse git blobs correctly This matches what we want for blobs. Trees are still in progress - we need a way to symlink to other objects, using that to determine ca. --- src/libutil/git.cc | 80 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 13 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 7e8f25b3e14..cf9b222ec80 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -36,20 +36,74 @@ void parseGit(ParseSink & sink, Source & source) { parse(sink, source, ""); } -static void parse(ParseSink & sink, Source & source, const Path & path) { - uint8_t buf[4]; - - std::basic_string_view buf_v { - (const uint8_t *) & buf, - std::size(buf) - }; - source(buf_v); - if (buf_v.compare((const uint8_t *)"blob")) { - uint8_t space; - source(& space, 1); - } - else { +string getStringUntil(Source & source, char byte) { + string s; + unsigned char n[1]; + source(n, 1); + while (*n != byte) { + s += *n; + source(n, 1); } + return s; +} + +string getString(Source & source, int n){ + std::vector v(n); + source(v.data(), n); + return std::string(v.begin(), v.end()); +} + +static void parse(ParseSink & sink, Source & source, const Path & path) { + auto type = getString(source, 5); + + if (type == "blob ") { + sink.createRegularFile(path); + + unsigned long long size = std::stoi(getStringUntil(source, 0)); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::vector buf(65536); + + while (left) { + checkInterrupt(); + auto n = buf.size(); + if ((unsigned long long)n > left) n = left; + source(buf.data(), n); + sink.receiveContents(buf.data(), n); + left -= n; + } + } else if (type == "tree ") { + unsigned long long size = std::stoi(getStringUntil(source, 0)); + unsigned long long left = size; + + sink.createDirectory(path); + + while (left) { + string perms = getStringUntil(source, ' '); + left -= perms.size(); + left -= 1; + + int perm = std::stoi(perms); + if (perm != 100644 && perm != 100755 && perm != 644 && perm != 755 && perm != 40000) + throw Error(format("Unknown Git permission: %d") % perm); + + // TODO: handle permissions somehow + + string name = getStringUntil(source, 0); + left -= name.size(); + left -= 1; + + string hashs = getString(source, 20); + left -= 20; + + Hash hash(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); + + sink.createSymlink(path + "/" + name, "../" + name); + } + } else throw Error("input doesn't look like a Git object"); } // TODO stream file into sink, rather than reading into vector From f292e50df72356a84d103e38e1447882d01dfca4 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 17:24:01 -0500 Subject: [PATCH 03/35] Add --git argument to add-to-store --- src/nix/add-to-store.cc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f43f774c1c8..c5b7c0368f7 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "store-api.hh" #include "archive.hh" +#include "git.hh" using namespace nix; @@ -9,6 +10,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; + bool git = false; CmdAddToStore() { @@ -21,6 +23,12 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "git", + .description = "treat path as a git object", + .handler = {&this->git, true}, + }); } std::string description() override @@ -40,15 +48,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand { if (!namePart) namePart = baseNameOf(path); + auto ingestionMethod = git ? FileIngestionMethod::Git : FileIngestionMethod::Recursive; + StringSink sink; - dumpPath(path, sink); + if (git) + dumpGit(path, sink); + else + dumpPath(path, sink); auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); + ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart)); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = makeFixedOutputCA(ingestionMethod, info.narHash); if (!dryRun) { auto source = StringSource { *sink.s }; From 1592d09a1fae82c6d7bb723f26b2429d427db9c3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 17:24:14 -0500 Subject: [PATCH 04/35] Add test for nix add-to-store --- tests/add.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/add.sh b/tests/add.sh index e26e05843d7..5ae4b1886e2 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -26,3 +26,7 @@ hash2=$(nix-hash --type sha256 --base32 ./dummy) echo $hash2 test "$hash1" = "sha256:$hash2" + +path5=$(nix add-to-store --git ./dummy) +hash3=$(nix-store -q --hash $path5) +test "$hash3" = "sha256:$(nix hash-git --base32 ./dummy)" From cf819912b40c4dea8c16c80b319108745de6b9f7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 28 May 2020 18:24:45 -0500 Subject: [PATCH 05/35] Depend on install for installcheck --- mk/tests.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/tests.mk b/mk/tests.mk index 70c30661b95..c3c1f4c28dd 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -8,7 +8,7 @@ define run-install-test endef # Color code from https://unix.stackexchange.com/a/10065 -installcheck: +installcheck: install @total=0; failed=0; \ red=""; \ green=""; \ From 9e9c2ce71f3cc4688a45eb948769683be84ceece Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 11:48:09 -0500 Subject: [PATCH 06/35] =?UTF-8?q?Give=20error=20when=20git=20ingestion=20d?= =?UTF-8?q?oesn=E2=80=99t=20use=20sha1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libstore/store-api.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f880cd5034c..c0303de679b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -178,6 +178,9 @@ StorePath Store::makeFixedOutputPath( const StorePathSet & references, bool hasSelfReference) const { + if (method == FileIngestionMethod::Git && hash.type != htSHA1) + throw Error("Git file ingestion must use sha1 hash"); + if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { From 6f7fab93580dc093d24285205fd7a969c8962940 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 12:24:05 -0500 Subject: [PATCH 07/35] Use SHA1 for Git objects This is really bad and dangerous! But Git migration to sha256 is still a ways away: https://lwn.net/Articles/811068/ So we need to allow it for the time being. --- src/libstore/daemon.cc | 11 ++++++++++- src/libstore/local-store.cc | 6 ++++-- src/nix/add-to-store.cc | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index aa2cbe6d487..7947e3f9708 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -708,11 +708,20 @@ static void performOp(TunnelLogger * logger, ref store, auto deriver = readString(from); if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash(readString(from), htSHA256); + + auto narHashString = readString(from); + info.references = readStorePaths(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); from >> info.ca >> repair >> dontCheckSigs; + + // git hashes are still using sha1 + if (hasPrefix(info.ca, "fixed:git:sha1:")) + info.narHash = Hash(narHashString, htSHA1); + else + info.narHash = Hash(narHashString, htSHA256); + if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e214c334b73..b4a66b445ad 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -909,7 +909,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash.type == htSHA256 || (i.narHash.type == htSHA1 && hasPrefix(i.ca, "fixed:git:"))); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1006,7 +1006,9 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr hashSink; - if (info.ca == "" || !info.references.count(info.path)) + if (hasPrefix(info.ca, "fixed:git:sha1:")) + hashSink = std::make_unique(htSHA1); // git objects use sha1 + else if (info.ca == "" || !info.references.count(info.path)) hashSink = std::make_unique(htSHA256); else hashSink = std::make_unique(htSHA256, storePathToHash(printStorePath(info.path))); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index c5b7c0368f7..efa223d9799 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -56,7 +56,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand else dumpPath(path, sink); - auto narHash = hashString(htSHA256, *sink.s); + auto narHash = hashString(git ? htSHA1 : htSHA256, *sink.s); ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart)); info.narHash = narHash; From ee989c62b726b7edaa9a161cf421078b037c7eca Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 12:27:48 -0500 Subject: [PATCH 08/35] Pass storeDir to restoreGit We need access to other things in the store. This is kind of dangerous though if things are added in the wrong order. --- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 6 +++--- src/libutil/git.cc | 14 ++++++-------- src/libutil/git.hh | 4 ++-- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 7947e3f9708..6fd582624eb 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -385,7 +385,7 @@ static void performOp(TunnelLogger * logger, ref store, } case FileIngestionMethod::Git: { ParseSink sink; - parseGit(sink, savedNAR); + parseGit(sink, savedNAR, store->storeDir); break; } case FileIngestionMethod::Flat: { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b4a66b445ad..6f5a3517e5f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1020,8 +1020,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, }); if (hasPrefix(info.ca, "fixed:git:")) - restoreGit(realPath, wrapperSource); - else + restoreGit(realPath, wrapperSource, realStoreDir); + else restorePath(realPath, wrapperSource); auto hashResult = hashSink->finish(); @@ -1084,7 +1084,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam } case FileIngestionMethod::Git: { StringSource source(dump); - restoreGit(realPath, source); + restoreGit(realPath, source, realStoreDir); break; } } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index cf9b222ec80..5581c4f0a7c 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -21,19 +21,17 @@ using namespace std::string_literals; namespace nix { -static void parse(ParseSink & sink, Source & source, const Path & path); +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir); // Converts a Path to a ParseSink -void restoreGit(const Path & path, Source & source) { - +void restoreGit(const Path & path, Source & source, const Path & storeDir) { RestoreSink sink; sink.dstPath = path; - parseGit(sink, source); - + parseGit(sink, source, storeDir); } -void parseGit(ParseSink & sink, Source & source) { - parse(sink, source, ""); +void parseGit(ParseSink & sink, Source & source, const Path & storeDir) { + parse(sink, source, "", storeDir); } string getStringUntil(Source & source, char byte) { @@ -53,7 +51,7 @@ string getString(Source & source, int n){ return std::string(v.begin(), v.end()); } -static void parse(ParseSink & sink, Source & source, const Path & path) { +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir) { auto type = getString(source, 5); if (type == "blob ") { diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 7b11f72be29..aed3de28d5e 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -12,9 +12,9 @@ enum struct GitMode { Regular, }; -void restoreGit(const Path & path, Source & source); +void restoreGit(const Path & path, Source & source, const Path & storeDir); -void parseGit(ParseSink & sink, Source & source); +void parseGit(ParseSink & sink, Source & source, const Path & storeDir); // Dumps a single file to a sink GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink); From 9cd3ac727028a7a44b2396e3e20abc8dceff2352 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 12:57:30 -0500 Subject: [PATCH 09/35] Pad 0s for dumpGitInternal --- src/libutil/git.cc | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 5581c4f0a7c..febbb12c101 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -127,7 +127,7 @@ GitMode dumpGitTree(const GitTree & entries, Sink & sink) case GitMode::Executable: mode = 100755; break; case GitMode::Regular: mode = 100644; break; } - s1 += (format("%6d %s\0%s"s) % mode % i.first % i.second.second.hash).str(); + s1 += (format("%06d %s\0%s"s) % mode % i.first % i.second.second.hash).str(); } std::string s2 = (format("tree %d\0%s"s) % s1.size() % s1).str(); @@ -138,39 +138,45 @@ GitMode dumpGitTree(const GitTree & entries, Sink & sink) return GitMode::Directory; } -// Returns the perm in addition -std::pair dumpGitHashInternal( - std::function()> makeHashSink, - const Path & path, PathFilter & filter) +static std::pair dumpGitHashInternal(HashType ht, const Path & path, PathFilter & filter); + +static GitMode dumpGitInternal(HashType ht, const Path & path, Sink & sink, PathFilter & filter) { - auto hashSink = makeHashSink(); struct stat st; GitMode perm; if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path '%1%'") % path); - if (S_ISREG(st.st_mode)) { - perm = dumpGitBlob(path, st, *hashSink); - } else if (S_ISDIR(st.st_mode)) { + if (S_ISREG(st.st_mode)) + perm = dumpGitBlob(path, st, sink); + else if (S_ISDIR(st.st_mode)) { GitTree entries; for (auto & i : readDirectory(path)) - if (filter(path + "/" + i.name)) { - entries[i.name] = dumpGitHashInternal(makeHashSink, path + "/" + i.name, filter); - } - perm = dumpGitTree(entries, *hashSink); - } else { - throw Error(format("file '%1%' has an unsupported type") % path); - } + if (filter(path + "/" + i.name)) + entries[i.name] = dumpGitHashInternal(ht, path + "/" + i.name, filter); + perm = dumpGitTree(entries, sink); + } else throw Error(format("file '%1%' has an unsupported type") % path); + + return perm; +} + +static std::pair dumpGitHashInternal(HashType ht, const Path & path, PathFilter & filter) +{ + auto hashSink = new HashSink(ht); + auto perm = dumpGitInternal(ht, path, *hashSink, filter); auto hash = hashSink->finish().first; return std::pair { perm, hash }; } -Hash dumpGitHash( - std::function()> makeHashSink, - const Path & path, PathFilter & filter) +Hash dumpGitHash(HashType ht, const Path & path, PathFilter & filter) +{ + return dumpGitHashInternal(ht, path, filter).second; +} + +void dumpGit(HashType ht, const Path & path, Sink & sink, PathFilter & filter) { - return dumpGitHashInternal(makeHashSink, path, filter).second; + dumpGitInternal(ht, path, sink, filter); } } From b5ed6a9c8b05c8c0af78c2d907a668d0571b3b89 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 13:39:00 -0500 Subject: [PATCH 10/35] Add createExecutableFile primitive to ParseSink This is needed to create files based on git permissions --- src/libutil/fs-sink.hh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index d3d1e43b79f..5ac49d83d41 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -13,6 +13,7 @@ struct ParseSink virtual void createDirectory(const Path & path) { }; virtual void createRegularFile(const Path & path) { }; + virtual void createExecutableFile(const Path & path) { }; virtual void isExecutable() { }; virtual void preallocateContents(unsigned long long size) { }; virtual void receiveContents(unsigned char * data, unsigned int len) { }; @@ -39,6 +40,13 @@ struct RestoreSink : ParseSink if (!fd) throw SysError(format("creating file '%1%'") % p); } + void createExecutableFile(const Path & path) + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0777); + if (!fd) throw SysError(format("creating file '%1%'") % p); + } + void isExecutable() { struct stat st; From 6ceba63920ab7c2057764afb9bebcd6b3bd7268a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 14:06:02 -0500 Subject: [PATCH 11/35] Read related git objects from nix store in parseGit --- src/libutil/git.cc | 53 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index febbb12c101..726248cce97 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -87,8 +87,6 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa if (perm != 100644 && perm != 100755 && perm != 644 && perm != 755 && perm != 40000) throw Error(format("Unknown Git permission: %d") % perm); - // TODO: handle permissions somehow - string name = getStringUntil(source, 0); left -= name.size(); left -= 1; @@ -96,10 +94,53 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa string hashs = getString(source, 20); left -= 20; - Hash hash(htSHA1); - std::copy(hashs.begin(), hashs.end(), hash.hash); - - sink.createSymlink(path + "/" + name, "../" + name); + Hash hash1(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash1.hash); + + Hash hash2 = hashString(htSHA256, "fixed:out:git:" + hash1.to_string(Base16) + ":"); + Hash hash3 = hashString(htSHA256, "output:out:" + hash2.to_string(Base16) + ":" + storeDir + ":" + name); + Hash hash4 = compressHash(hash3, 20); + + string entryName = hash4.to_string(Base32, false) + "-" + name; + Path entry = storeDir + "/" + entryName; + + struct stat st; + if (lstat(entry.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % entry); + + if (S_ISREG(st.st_mode)) { + if (perm == 40000) + throw SysError(format("file is a file but expected to be a directory '%1%'") % entry); + + if (perm == 100755 || perm == 755) + sink.createExecutableFile(path + "/" + name); + else + sink.createRegularFile(path + "/" + name); + + unsigned long long size = st.st_size; + sink.preallocateContents(size); + + unsigned long long left = size; + std::vector buf(65536); + + StringSink ssink; + readFile(entry, ssink); + AutoCloseFD fd = open(entry.c_str(), O_RDONLY | O_CLOEXEC); + + while (left) { + checkInterrupt(); + auto n = buf.size(); + if ((unsigned long long)n > left) n = left; + ssink(buf.data(), n); + sink.receiveContents(buf.data(), n); + left -= n; + } + } else if (S_ISDIR(st.st_mode)) { + if (perm != 40000) + throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); + + sink.createSymlink(path + "/" + name, "../" + entryName); + } else throw Error(format("file '%1%' has an unsupported type") % entry); } } else throw Error("input doesn't look like a Git object"); } From e44956d8298dba7a1656408d2b454cd94822b9d1 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 15:07:34 -0500 Subject: [PATCH 12/35] Allow sha1 in --hash results This updates the remote protocol to try to handle sha1 hashes. --- src/libstore/remote-store.cc | 7 ++++++- src/nix-store/nix-store.cc | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 842c2e17666..16408bb5907 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -375,13 +375,18 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, info = std::make_shared(path.clone()); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->narHash = Hash(readString(conn->from), htSHA256); + + auto narHashString = readString(conn->from); + info->references = readStorePaths(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); conn->from >> info->ca; + info->narHash = Hash(narHashString, hasPrefix(info->ca, "fixed:git:") ? htSHA1 : htSHA256); + } else { + info->narHash = Hash(narHashString, htSHA256); } } callback(std::move(info)); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 68fe6f72735..5b4ac6a5e71 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -372,7 +372,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); + assert(info->narHash.type == htSHA256 || (info->narHash.type == htSHA1 && hasPrefix(info->ca, "fixed:git:"))); cout << fmt("%s\n", info->narHash.to_string(Base32)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); From 646862eeebe0c98820f9d21c08a0c1e4506d05f2 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 15:08:09 -0500 Subject: [PATCH 13/35] Add test for adding git tree hash --- tests/add.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/add.sh b/tests/add.sh index 5ae4b1886e2..f9fe2475a7f 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -30,3 +30,10 @@ test "$hash1" = "sha256:$hash2" path5=$(nix add-to-store --git ./dummy) hash3=$(nix-store -q --hash $path5) test "$hash3" = "sha256:$(nix hash-git --base32 ./dummy)" + +mkdir -p dummy2 +echo hello > dummy2/hello +nix add-to-store --git ./dummy2/hello +path6=$(nix add-to-store --git ./dummy2) +hash4=$(nix-store -q --hash $path6) +test "$hash4" = "sha256:$(nix hash-git --base32 ./dummy2)" From 592e92664d670db7b7def000b544654e1893b47b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 15:25:49 -0500 Subject: [PATCH 14/35] Expand git tests --- tests/add.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/add.sh b/tests/add.sh index f9fe2475a7f..0c20f0dfff8 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -27,13 +27,26 @@ echo $hash2 test "$hash1" = "sha256:$hash2" +# Git tests + path5=$(nix add-to-store --git ./dummy) hash3=$(nix-store -q --hash $path5) -test "$hash3" = "sha256:$(nix hash-git --base32 ./dummy)" +test "$hash3" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy)" +rm -rf dummy2 mkdir -p dummy2 echo hello > dummy2/hello nix add-to-store --git ./dummy2/hello path6=$(nix add-to-store --git ./dummy2) hash4=$(nix-store -q --hash $path6) -test "$hash4" = "sha256:$(nix hash-git --base32 ./dummy2)" +test "$hash4" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy2)" + +rm -rf dummy3 +mkdir -p dummy3 +mkdir -p dummy3/hello +echo hello > dummy3/hello/hello +nix add-to-store --git ./dummy3/hello/hello +nix add-to-store --git ./dummy3/hello +path7=$(nix add-to-store --git ./dummy3) +hash5=$(nix-store -q --hash $path7) +test "$hash5" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy3)" From a4a038fd8a3fd8765f0b3b670bb64658de910006 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 17:14:40 -0500 Subject: [PATCH 15/35] Cleanup git.cc code --- src/libutil/git.cc | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 726248cce97..ef3fdfdd8ea 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -30,11 +30,13 @@ void restoreGit(const Path & path, Source & source, const Path & storeDir) { parseGit(sink, source, storeDir); } -void parseGit(ParseSink & sink, Source & source, const Path & storeDir) { +void parseGit(ParseSink & sink, Source & source, const Path & storeDir) +{ parse(sink, source, "", storeDir); } -string getStringUntil(Source & source, char byte) { +static string getStringUntil(Source & source, char byte) +{ string s; unsigned char n[1]; source(n, 1); @@ -45,13 +47,25 @@ string getStringUntil(Source & source, char byte) { return s; } -string getString(Source & source, int n){ +static string getString(Source & source, int n) +{ std::vector v(n); source(v.data(), n); return std::string(v.begin(), v.end()); } -static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir) { +// Unfortunately, no access to libstore headers here. +static Path getStorePath(const Path & storeDir, Hash hash, string name) +{ + Hash hash1 = hashString(htSHA256, "fixed:out:git:" + hash.to_string(Base16) + ":"); + Hash hash2 = hashString(htSHA256, "output:out:" + hash1.to_string(Base16) + ":" + storeDir + ":" + name); + Hash hash3 = compressHash(hash2, 20); + + return storeDir + "/" + hash3.to_string(Base32, false) + "-" + name; +} + +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir) +{ auto type = getString(source, 5); if (type == "blob ") { @@ -94,15 +108,10 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa string hashs = getString(source, 20); left -= 20; - Hash hash1(htSHA1); - std::copy(hashs.begin(), hashs.end(), hash1.hash); - - Hash hash2 = hashString(htSHA256, "fixed:out:git:" + hash1.to_string(Base16) + ":"); - Hash hash3 = hashString(htSHA256, "output:out:" + hash2.to_string(Base16) + ":" + storeDir + ":" + name); - Hash hash4 = compressHash(hash3, 20); + Hash hash(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); - string entryName = hash4.to_string(Base32, false) + "-" + name; - Path entry = storeDir + "/" + entryName; + Path entry = getStorePath(storeDir, hash, name); struct stat st; if (lstat(entry.c_str(), &st)) @@ -139,7 +148,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa if (perm != 40000) throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); - sink.createSymlink(path + "/" + name, "../" + entryName); + sink.createSymlink(path + "/" + name, entry); } else throw Error(format("file '%1%' has an unsupported type") % entry); } } else throw Error("input doesn't look like a Git object"); From 4ecb49d61c11fc4ea0f67a5a47dc641f58d1f3dd Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 18:02:08 -0500 Subject: [PATCH 16/35] Use path-based addToStore method for add-to-store This is more direct. --- src/nix/add-to-store.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index efa223d9799..c224d9b2223 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -63,10 +63,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(ingestionMethod, info.narHash); - if (!dryRun) { - auto source = StringSource { *sink.s }; - store->addToStore(info, source); - } + if (!dryRun) + store->addToStore(*namePart, path, ingestionMethod, git ? htSHA1 : htSHA256); logger->stdout("%s", store->printStorePath(info.path)); } From d68b774cc8fd1fadab76dc5b5eb621194cf9a627 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 18:26:18 -0500 Subject: [PATCH 17/35] Properly handle realStoreDir vs. storeDir in git.cc We need both to properly mess with the file system. storeDir goes into the hash while realStoreDir is what we read & write to. --- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/libutil/git.cc | 18 +++++++++--------- src/libutil/git.hh | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6fd582624eb..7270ae6cbc6 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -385,7 +385,7 @@ static void performOp(TunnelLogger * logger, ref store, } case FileIngestionMethod::Git: { ParseSink sink; - parseGit(sink, savedNAR, store->storeDir); + parseGit(sink, savedNAR, store->storeDir, store->storeDir); break; } case FileIngestionMethod::Flat: { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6f5a3517e5f..2c624d6460d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1020,7 +1020,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, }); if (hasPrefix(info.ca, "fixed:git:")) - restoreGit(realPath, wrapperSource, realStoreDir); + restoreGit(realPath, wrapperSource, realStoreDir, storeDir); else restorePath(realPath, wrapperSource); @@ -1084,7 +1084,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam } case FileIngestionMethod::Git: { StringSource source(dump); - restoreGit(realPath, source, realStoreDir); + restoreGit(realPath, source, realStoreDir, storeDir); break; } } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index ef3fdfdd8ea..b876962a43c 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -21,18 +21,18 @@ using namespace std::string_literals; namespace nix { -static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir); +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir); // Converts a Path to a ParseSink -void restoreGit(const Path & path, Source & source, const Path & storeDir) { +void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir) { RestoreSink sink; sink.dstPath = path; - parseGit(sink, source, storeDir); + parseGit(sink, source, realStoreDir, storeDir); } -void parseGit(ParseSink & sink, Source & source, const Path & storeDir) +void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir) { - parse(sink, source, "", storeDir); + parse(sink, source, "", realStoreDir, storeDir); } static string getStringUntil(Source & source, char byte) @@ -55,16 +55,16 @@ static string getString(Source & source, int n) } // Unfortunately, no access to libstore headers here. -static Path getStorePath(const Path & storeDir, Hash hash, string name) +static Path getStorePath(const Path & realStoreDir, const Path & storeDir, Hash hash, string name) { Hash hash1 = hashString(htSHA256, "fixed:out:git:" + hash.to_string(Base16) + ":"); Hash hash2 = hashString(htSHA256, "output:out:" + hash1.to_string(Base16) + ":" + storeDir + ":" + name); Hash hash3 = compressHash(hash2, 20); - return storeDir + "/" + hash3.to_string(Base32, false) + "-" + name; + return realStoreDir + "/" + hash3.to_string(Base32, false) + "-" + name; } -static void parse(ParseSink & sink, Source & source, const Path & path, const Path & storeDir) +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir) { auto type = getString(source, 5); @@ -111,7 +111,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa Hash hash(htSHA1); std::copy(hashs.begin(), hashs.end(), hash.hash); - Path entry = getStorePath(storeDir, hash, name); + Path entry = getStorePath(realStoreDir, storeDir, hash, name); struct stat st; if (lstat(entry.c_str(), &st)) diff --git a/src/libutil/git.hh b/src/libutil/git.hh index aed3de28d5e..1d773d3750f 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -12,9 +12,9 @@ enum struct GitMode { Regular, }; -void restoreGit(const Path & path, Source & source, const Path & storeDir); +void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir); -void parseGit(ParseSink & sink, Source & source, const Path & storeDir); +void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir); // Dumps a single file to a sink GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink); From 18d4a3d5f37a7f24928a7a0dfd6abb92eec2c95d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 18:52:31 -0500 Subject: [PATCH 18/35] Allow sending git objects over daemon --- src/libstore/daemon.cc | 2 +- src/libstore/remote-store.cc | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 7270ae6cbc6..4bac5bff68b 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -398,7 +398,7 @@ static void performOp(TunnelLogger * logger, ref store, if (!savedRegular.regular) throw Error("regular file expected"); auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, + method == FileIngestionMethod::Flat ? savedRegular.s : *savedNAR.data, baseName, method, hashAlgo); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 16408bb5907..019d3e6f02d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -8,6 +8,7 @@ #include "derivations.hh" #include "pool.hh" #include "finally.hh" +#include "git.hh" #include #include @@ -493,7 +494,6 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - if (method == FileIngestionMethod::Git) throw Error("cannot remotely add to store using the git file ingestion method"); auto conn(getConnection()); @@ -512,7 +512,10 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - dumpPath(srcPath, conn->to, filter); + if (method == FileIngestionMethod::Git) + dumpGit(hashAlgo, srcPath, conn->to, filter); + else + dumpPath(srcPath, conn->to, filter); } conn->to.warn = false; conn.processStderr(); From d8f34fd166b60848b7614db1e8f8b69799b7947c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 18:52:50 -0500 Subject: [PATCH 19/35] Recursively add to store from git ingestion --- src/libstore/local-store.cc | 11 ++++++++++- src/libstore/remote-store.cc | 13 +++++++++++-- tests/add.sh | 3 --- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2c624d6460d..13c7109c31f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1131,7 +1131,16 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, case FileIngestionMethod::Recursive: dumpPath(srcPath, sink, filter); break; - case FileIngestionMethod::Git:{ + } + case FileIngestionMethod::Git: { + // recursively add to store if path is a directory + struct stat st; + if (lstat(srcPath.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % srcPath); + if (S_ISDIR(st.st_mode)) + for (auto & i : readDirectory(srcPath)) + addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair); + dumpGit(srcPath, sink, filter); break; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 019d3e6f02d..b814ed67865 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -494,10 +494,19 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + Path srcPath(absPath(_srcPath)); - auto conn(getConnection()); + // recursively add to store if path is a directory + if (method == FileIngestionMethod::Git) { + struct stat st; + if (lstat(srcPath.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % srcPath); + if (S_ISDIR(st.st_mode)) + for (auto & i : readDirectory(srcPath)) + addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair); + } - Path srcPath(absPath(_srcPath)); + auto conn(getConnection()); conn->to << wopAddToStore diff --git a/tests/add.sh b/tests/add.sh index 0c20f0dfff8..389bdb57245 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -36,7 +36,6 @@ test "$hash3" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy)" rm -rf dummy2 mkdir -p dummy2 echo hello > dummy2/hello -nix add-to-store --git ./dummy2/hello path6=$(nix add-to-store --git ./dummy2) hash4=$(nix-store -q --hash $path6) test "$hash4" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy2)" @@ -45,8 +44,6 @@ rm -rf dummy3 mkdir -p dummy3 mkdir -p dummy3/hello echo hello > dummy3/hello/hello -nix add-to-store --git ./dummy3/hello/hello -nix add-to-store --git ./dummy3/hello path7=$(nix add-to-store --git ./dummy3) hash5=$(nix-store -q --hash $path7) test "$hash5" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy3)" From 27ffbc7c1425c1091db8fb73e37e2e1e2c3495fa Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 23:02:17 -0500 Subject: [PATCH 20/35] Add hash to local store correctly from dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Need to use the htSHA1 we were given, don’t recompute. --- src/libstore/local-store.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 13c7109c31f..5e6b4b94092 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1096,7 +1096,12 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (method == FileIngestionMethod::Recursive) { + if (method == FileIngestionMethod::Git) { + if (hashAlgo != htSHA1) + throw Error("Git must use sha1 hash"); + hash.first = h; + hash.second = dump.size(); + } else if (method == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else From 05cbfdef9f76cafd5093dd08a8bfdea85f5bb297 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 23:07:50 -0500 Subject: [PATCH 21/35] Move git tests to own file this makes maintanence easier --- tests/add.sh | 21 --------------------- tests/git.sh | 36 ++++++++++++++++++++++++++++++++++++ tests/hash.sh | 15 --------------- tests/local.mk | 3 ++- 4 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 tests/git.sh diff --git a/tests/add.sh b/tests/add.sh index 389bdb57245..e26e05843d7 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -26,24 +26,3 @@ hash2=$(nix-hash --type sha256 --base32 ./dummy) echo $hash2 test "$hash1" = "sha256:$hash2" - -# Git tests - -path5=$(nix add-to-store --git ./dummy) -hash3=$(nix-store -q --hash $path5) -test "$hash3" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy)" - -rm -rf dummy2 -mkdir -p dummy2 -echo hello > dummy2/hello -path6=$(nix add-to-store --git ./dummy2) -hash4=$(nix-store -q --hash $path6) -test "$hash4" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy2)" - -rm -rf dummy3 -mkdir -p dummy3 -mkdir -p dummy3/hello -echo hello > dummy3/hello/hello -path7=$(nix add-to-store --git ./dummy3) -hash5=$(nix-store -q --hash $path7) -test "$hash5" = "sha1:$(nix hash-git --type sha1 --base32 ./dummy3)" diff --git a/tests/git.sh b/tests/git.sh new file mode 100644 index 00000000000..08ec310e5af --- /dev/null +++ b/tests/git.sh @@ -0,0 +1,36 @@ +source common.sh + +try () { + hash=$(nix hash-git --base16 --type sha1 $TEST_ROOT/hash-path) + if test "$hash" != "$1"; then + echo "git hash, expected $1, got $hash" + exit 1 + fi +} + +rm -rf $TEST_ROOT/hash-path +mkdir $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path/hello + +try "117c62a8c5e01758bd284126a6af69deab9dbbe2" + +rm -rf $TEST_ROOT/dummy1 +echo Hello World! > $TEST_ROOT/dummy1 +path1=$(nix add-to-store --git $TEST_ROOT/dummy1) +hash1=$(nix-store -q --hash $path1) +test "$hash1" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy1)" + +rm -rf $TEST_ROOT/dummy2 +mkdir -p $TEST_ROOT/dummy2 +echo hello > $TEST_ROOT/dummy2/hello +path2=$(nix add-to-store --git $TEST_ROOT/dummy2) +hash2=$(nix-store -q --hash $path2) +test "$hash2" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy2)" + +rm -rf $TEST_ROOT/dummy3 +mkdir -p $TEST_ROOT/dummy3 +mkdir -p $TEST_ROOT/dummy3/hello +echo hello > $TEST_ROOT/dummy3/hello/hello +path3=$(nix add-to-store --git $TEST_ROOT/dummy3) +hash3=$(nix-store -q --hash $path3) +test "$hash3" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy3)" diff --git a/tests/hash.sh b/tests/hash.sh index 33b5bef7595..4cfc9790101 100644 --- a/tests/hash.sh +++ b/tests/hash.sh @@ -85,18 +85,3 @@ try3() { try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" - -# Git. -try4 () { - hash=$(nix hash-git --base16 --type sha1 $TEST_ROOT/hash-path) - if test "$hash" != "$1"; then - echo "git hash, expected $1, got $hash" - exit 1 - fi -} - -rm -rf $TEST_ROOT/hash-path -mkdir $TEST_ROOT/hash-path -echo "Hello World" > $TEST_ROOT/hash-path/hello - -try4 "117c62a8c5e01758bd284126a6af69deab9dbbe2" diff --git a/tests/local.mk b/tests/local.mk index 56e5640ca33..35a80a16a5a 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -30,7 +30,8 @@ nix_tests = \ nix-copy-ssh.sh \ post-hook.sh \ function-trace.sh \ - recursive.sh + recursive.sh \ + git.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) From 97ce2cc5d3d83d33f6056d125f1a41ca46af07d4 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 29 May 2020 23:53:02 -0500 Subject: [PATCH 22/35] Revert "Depend on install for installcheck" This reverts commit 7b6186dc661c036bb251ba3158622120acaebc3f. --- mk/tests.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/tests.mk b/mk/tests.mk index c3c1f4c28dd..70c30661b95 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -8,7 +8,7 @@ define run-install-test endef # Color code from https://unix.stackexchange.com/a/10065 -installcheck: install +installcheck: @total=0; failed=0; \ red=""; \ green=""; \ From 229ce9c454cf90344ebdecc3d43219395fdfe9cf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 10:28:09 -0500 Subject: [PATCH 23/35] Make computeStorePathForPath work in Git ingestion --- src/libstore/store-api.cc | 18 +++++++++++++++--- src/libutil/hash.cc | 8 ++++++++ src/libutil/hash.hh | 3 +++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c0303de679b..4f99cfe2fa4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -209,9 +209,21 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, std::pair Store::computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const { - Hash h = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath, filter).first - : hashFile(hashAlgo, srcPath); + Hash h; + switch (method) { + case FileIngestionMethod::Recursive: { + h = hashPath(hashAlgo, srcPath, filter).first; + break; + } + case FileIngestionMethod::Git: { + h = hashGit(hashAlgo, srcPath, filter).first; + break; + } + case FileIngestionMethod::Flat: { + h = hashFile(hashAlgo, srcPath); + break; + } + } return std::make_pair(makeFixedOutputPath(method, h, name), h); } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 7caee1da7f8..b3f3db23178 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -6,6 +6,7 @@ #include "hash.hh" #include "archive.hh" +#include "git.hh" #include "util.hh" #include "istringstream_nocopy.hh" @@ -307,6 +308,13 @@ HashResult hashPath( return sink.finish(); } +HashResult hashGit( + HashType ht, const Path & path, PathFilter & filter) +{ + HashSink sink(ht); + dumpGit(ht, path, sink, filter); + return sink.finish(); +} Hash compressHash(const Hash & hash, unsigned int newSize) { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ea9fca3e74f..1af94c2b47c 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -110,6 +110,9 @@ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); +HashResult hashGit(HashType ht, const Path & path, + PathFilter & filter = defaultPathFilter); + /* Compress a hash to the specified number of bytes by cyclically XORing bytes together. */ Hash compressHash(const Hash & hash, unsigned int newSize); From 031fa72c20c2c5cd4ed996025df1ad44ea134b57 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 11:05:09 -0500 Subject: [PATCH 24/35] Use SHA256 for narHash narHash is the hash of the nar, not the git objects. --- src/libstore/daemon.cc | 11 +---------- src/libstore/local-store.cc | 17 +++++------------ src/libstore/remote-store.cc | 9 ++------- src/libstore/store-api.cc | 4 +++- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 8 ++++---- tests/git.sh | 6 +++--- 7 files changed, 19 insertions(+), 38 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4bac5bff68b..117ee8c3585 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -708,20 +708,11 @@ static void performOp(TunnelLogger * logger, ref store, auto deriver = readString(from); if (deriver != "") info.deriver = store->parseStorePath(deriver); - - auto narHashString = readString(from); - + info.narHash = Hash(readString(from), htSHA256); info.references = readStorePaths(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); from >> info.ca >> repair >> dontCheckSigs; - - // git hashes are still using sha1 - if (hasPrefix(info.ca, "fixed:git:sha1:")) - info.narHash = Hash(narHashString, htSHA1); - else - info.narHash = Hash(narHashString, htSHA256); - if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5e6b4b94092..031ef367cae 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -909,7 +909,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256 || (i.narHash.type == htSHA1 && hasPrefix(i.ca, "fixed:git:"))); + assert(i.narHash.type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1006,9 +1006,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr hashSink; - if (hasPrefix(info.ca, "fixed:git:sha1:")) - hashSink = std::make_unique(htSHA1); // git objects use sha1 - else if (info.ca == "" || !info.references.count(info.path)) + if (info.ca == "" || !info.references.count(info.path)) hashSink = std::make_unique(htSHA256); else hashSink = std::make_unique(htSHA256, storePathToHash(printStorePath(info.path))); @@ -1051,7 +1049,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { - Hash h = hashString(hashAlgo, dump); + Hash h = hashString(method == FileIngestionMethod::Git ? htSHA1 : hashAlgo, dump); auto dstPath = makeFixedOutputPath(method, h, name); @@ -1096,12 +1094,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (method == FileIngestionMethod::Git) { - if (hashAlgo != htSHA1) - throw Error("Git must use sha1 hash"); - hash.first = h; - hash.second = dump.size(); - } else if (method == FileIngestionMethod::Recursive) { + if (method == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else @@ -1146,7 +1139,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, for (auto & i : readDirectory(srcPath)) addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair); - dumpGit(srcPath, sink, filter); + dumpGit(htSHA1, srcPath, sink, filter); break; } case FileIngestionMethod::Flat: { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b814ed67865..dbc70b5082b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -376,18 +376,13 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, info = std::make_shared(path.clone()); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - - auto narHashString = readString(conn->from); - + info->narHash = Hash(readString(conn->from), htSHA256); info->references = readStorePaths(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); conn->from >> info->ca; - info->narHash = Hash(narHashString, hasPrefix(info->ca, "fixed:git:") ? htSHA1 : htSHA256); - } else { - info->narHash = Hash(narHashString, htSHA256); } } callback(std::move(info)); @@ -522,7 +517,7 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { Finally cleanup([&]() { connections->decCapacity(); }); if (method == FileIngestionMethod::Git) - dumpGit(hashAlgo, srcPath, conn->to, filter); + dumpGit(htSHA1, srcPath, conn->to, filter); else dumpPath(srcPath, conn->to, filter); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4f99cfe2fa4..e94ed8a9ea0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -216,7 +216,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, break; } case FileIngestionMethod::Git: { - h = hashGit(hashAlgo, srcPath, filter).first; + h = hashGit(htSHA1, srcPath, filter).first; break; } case FileIngestionMethod::Flat: { @@ -853,6 +853,8 @@ Strings ValidPathInfo::shortRefs() const std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) { + if (method == FileIngestionMethod::Git && hash.type != htSHA1) + throw Error("git file ingestion must use sha1 hashes"); return "fixed:" + ingestionMethodPrefix(method) + hash.to_string(); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b4ac6a5e71..68fe6f72735 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -372,7 +372,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256 || (info->narHash.type == htSHA1 && hasPrefix(info->ca, "fixed:git:"))); + assert(info->narHash.type == htSHA256); cout << fmt("%s\n", info->narHash.to_string(Base32)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index c224d9b2223..731951bd0f7 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -56,12 +56,12 @@ struct CmdAddToStore : MixDryRun, StoreCommand else dumpPath(path, sink); - auto narHash = hashString(git ? htSHA1 : htSHA256, *sink.s); + auto hash = hashString(git ? htSHA1 : htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart)); - info.narHash = narHash; + ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); + info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(ingestionMethod, info.narHash); + info.ca = makeFixedOutputCA(ingestionMethod, hash); if (!dryRun) store->addToStore(*namePart, path, ingestionMethod, git ? htSHA1 : htSHA256); diff --git a/tests/git.sh b/tests/git.sh index 08ec310e5af..bd5a4742249 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -18,14 +18,14 @@ rm -rf $TEST_ROOT/dummy1 echo Hello World! > $TEST_ROOT/dummy1 path1=$(nix add-to-store --git $TEST_ROOT/dummy1) hash1=$(nix-store -q --hash $path1) -test "$hash1" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy1)" +test "$hash1" = "sha256:1brffhvj2c0z6x8qismd43m0iy8dsgfmy10bgg9w11szway2wp9v" rm -rf $TEST_ROOT/dummy2 mkdir -p $TEST_ROOT/dummy2 echo hello > $TEST_ROOT/dummy2/hello path2=$(nix add-to-store --git $TEST_ROOT/dummy2) hash2=$(nix-store -q --hash $path2) -test "$hash2" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy2)" +test "$hash2" = "sha256:1wnx6ldp06c1riiyhgqvfmhzpm664066i5hdq5fdcf1wg19mz45i" rm -rf $TEST_ROOT/dummy3 mkdir -p $TEST_ROOT/dummy3 @@ -33,4 +33,4 @@ mkdir -p $TEST_ROOT/dummy3/hello echo hello > $TEST_ROOT/dummy3/hello/hello path3=$(nix add-to-store --git $TEST_ROOT/dummy3) hash3=$(nix-store -q --hash $path3) -test "$hash3" = "sha1:$(nix hash-git --type sha1 --base32 $TEST_ROOT/dummy3)" +test "$hash3" = "sha256:153908hlsd9bhcpqxsfrcvqwqmxbhrizzjzn4rnggh3blzzdzim4" From d49c87323654575c37ca9c522ccf88b1ab2b7930 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 13:39:53 -0500 Subject: [PATCH 25/35] Use correct narHash in add-to-store We need to dump path and dump git here so that the hashes match what is expected elsewhere. --- src/nix/add-to-store.cc | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 731951bd0f7..ced8704ef0a 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -51,15 +51,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto ingestionMethod = git ? FileIngestionMethod::Git : FileIngestionMethod::Recursive; StringSink sink; - if (git) - dumpGit(path, sink); - else - dumpPath(path, sink); + dumpPath(path, sink); - auto hash = hashString(git ? htSHA1 : htSHA256, *sink.s); + auto narHash = hashString(htSHA256, *sink.s); + auto hash = git ? dumpGitHash(htSHA1, path) : narHash; ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); - info.narHash = hashString(htSHA256, *sink.s); + info.narHash = narHash; info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(ingestionMethod, hash); From cd3ef3f6a9469528feed6010a2e20e14a039fd08 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 13:48:14 -0500 Subject: [PATCH 26/35] Throw error when hashAlgo != SHA1 on git ingestion This should always use sha1 hash type, but we want to make sure the caller knows that. So just throw an error instead of ignoring hashAlgo on Git ingestion. --- src/libstore/local-store.cc | 10 ++++++++-- src/libstore/remote-store.cc | 5 ++++- src/libstore/store-api.cc | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 031ef367cae..b742857ccbb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1049,7 +1049,10 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { - Hash h = hashString(method == FileIngestionMethod::Git ? htSHA1 : hashAlgo, dump); + if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) + throw Error("git ingestion must use sha1 hash"); + + Hash h = hashString(hashAlgo, dump); auto dstPath = makeFixedOutputPath(method, h, name); @@ -1121,6 +1124,9 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); + if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) + throw Error("git ingestion must use sha1 hash"); + /* Read the whole path into memory. This is not a very scalable method for very large paths, but `copyPath' is mainly used for small files. */ @@ -1139,7 +1145,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, for (auto & i : readDirectory(srcPath)) addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair); - dumpGit(htSHA1, srcPath, sink, filter); + dumpGit(hashAlgo, srcPath, sink, filter); break; } case FileIngestionMethod::Flat: { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index dbc70b5082b..fbc161cc92c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -489,6 +489,9 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) + throw Error("git ingestion must use sha1 hash"); + Path srcPath(absPath(_srcPath)); // recursively add to store if path is a directory @@ -517,7 +520,7 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { Finally cleanup([&]() { connections->decCapacity(); }); if (method == FileIngestionMethod::Git) - dumpGit(htSHA1, srcPath, conn->to, filter); + dumpGit(hashAlgo, srcPath, conn->to, filter); else dumpPath(srcPath, conn->to, filter); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e94ed8a9ea0..39c4009b5a0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -216,7 +216,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, break; } case FileIngestionMethod::Git: { - h = hashGit(htSHA1, srcPath, filter).first; + h = hashGit(hashAlgo, srcPath, filter).first; break; } case FileIngestionMethod::Flat: { From 4dae98e044294f363ce9b97afeab8914e2573151 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 14:02:53 -0500 Subject: [PATCH 27/35] =?UTF-8?q?Don=E2=80=99t=20include=20realStoreDir=20?= =?UTF-8?q?refs=20in=20git=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit symlinks should be relative so that they look like: ../s5c0hnz9qfnpnn1bszfxicgz21d1fam3-dummy3 instead of /build/nix-test/store/s5c0hnz9qfnpnn1bszfxicgz21d1fam3-dummy3 This way our hashes will work with any real store dir. Note that /nix/store is still embedded in the store entry function. --- src/libutil/git.cc | 9 +++++---- tests/git.sh | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index b876962a43c..3472e812cce 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -55,13 +55,13 @@ static string getString(Source & source, int n) } // Unfortunately, no access to libstore headers here. -static Path getStorePath(const Path & realStoreDir, const Path & storeDir, Hash hash, string name) +static string getStoreEntry(const Path & storeDir, Hash hash, string name) { Hash hash1 = hashString(htSHA256, "fixed:out:git:" + hash.to_string(Base16) + ":"); Hash hash2 = hashString(htSHA256, "output:out:" + hash1.to_string(Base16) + ":" + storeDir + ":" + name); Hash hash3 = compressHash(hash2, 20); - return realStoreDir + "/" + hash3.to_string(Base32, false) + "-" + name; + return hash3.to_string(Base32, false) + "-" + name; } static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir) @@ -111,7 +111,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa Hash hash(htSHA1); std::copy(hashs.begin(), hashs.end(), hash.hash); - Path entry = getStorePath(realStoreDir, storeDir, hash, name); + string entryName = getStoreEntry(storeDir, hash, name); + Path entry = realStoreDir + "/" + entryName; struct stat st; if (lstat(entry.c_str(), &st)) @@ -148,7 +149,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa if (perm != 40000) throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); - sink.createSymlink(path + "/" + name, entry); + sink.createSymlink(path + "/" + name, "../" + entryName); } else throw Error(format("file '%1%' has an unsupported type") % entry); } } else throw Error("input doesn't look like a Git object"); diff --git a/tests/git.sh b/tests/git.sh index bd5a4742249..d9daa9f2d77 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -33,4 +33,4 @@ mkdir -p $TEST_ROOT/dummy3/hello echo hello > $TEST_ROOT/dummy3/hello/hello path3=$(nix add-to-store --git $TEST_ROOT/dummy3) hash3=$(nix-store -q --hash $path3) -test "$hash3" = "sha256:153908hlsd9bhcpqxsfrcvqwqmxbhrizzjzn4rnggh3blzzdzim4" +test "$hash3" = "sha256:06psnck6rb12xyd8h4y2j801xf2lvhjwx9gxzxk25h922nm71lmf" From 1b14945a3bf741a44e520d156c01ca79a50975bb Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 15:16:30 -0500 Subject: [PATCH 28/35] Use FdSink to simplify copying --- src/libutil/fs-sink.hh | 7 +++++++ src/libutil/git.cc | 24 +++++------------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 5ac49d83d41..7d8764e8ba9 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -19,6 +19,8 @@ struct ParseSink virtual void receiveContents(unsigned char * data, unsigned int len) { }; virtual void createSymlink(const Path & path, const string & target) { }; + + virtual int getFD() { return 0; }; }; struct RestoreSink : ParseSink @@ -81,6 +83,11 @@ struct RestoreSink : ParseSink Path p = dstPath + path; nix::createSymlink(target, p); } + + int getFD() + { + return fd.get(); + } }; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 3472e812cce..f2be3eaad54 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -16,6 +16,7 @@ #include "hash.hh" #include "git.hh" +#include "serialise.hh" using namespace std::string_literals; @@ -112,7 +113,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa std::copy(hashs.begin(), hashs.end(), hash.hash); string entryName = getStoreEntry(storeDir, hash, name); - Path entry = realStoreDir + "/" + entryName; + Path entry = absPath(realStoreDir + "/" + entryName); struct stat st; if (lstat(entry.c_str(), &st)) @@ -127,24 +128,9 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa else sink.createRegularFile(path + "/" + name); - unsigned long long size = st.st_size; - sink.preallocateContents(size); - - unsigned long long left = size; - std::vector buf(65536); - - StringSink ssink; - readFile(entry, ssink); - AutoCloseFD fd = open(entry.c_str(), O_RDONLY | O_CLOEXEC); - - while (left) { - checkInterrupt(); - auto n = buf.size(); - if ((unsigned long long)n > left) n = left; - ssink(buf.data(), n); - sink.receiveContents(buf.data(), n); - left -= n; - } + sink.preallocateContents(st.st_size); + FdSink fsink(sink.getFD()); + readFile(entry, fsink); } else if (S_ISDIR(st.st_mode)) { if (perm != 40000) throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); From 54ee74e9b600c094bc2c69c5b67c502b30c88ee5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 15:17:09 -0500 Subject: [PATCH 29/35] Fixup bad hash in tests/git.sh --- tests/git.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/git.sh b/tests/git.sh index d9daa9f2d77..f8890ce50e2 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -25,7 +25,7 @@ mkdir -p $TEST_ROOT/dummy2 echo hello > $TEST_ROOT/dummy2/hello path2=$(nix add-to-store --git $TEST_ROOT/dummy2) hash2=$(nix-store -q --hash $path2) -test "$hash2" = "sha256:1wnx6ldp06c1riiyhgqvfmhzpm664066i5hdq5fdcf1wg19mz45i" +test "$hash2" = "sha256:1pgyz59p65wd11vfxp3vi673ijwjfg7i4ynlqlsgzvg9dvh67dpj" rm -rf $TEST_ROOT/dummy3 mkdir -p $TEST_ROOT/dummy3 From b1e1ace44934fc847abb46586396d41d6acfc66d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 15:32:27 -0500 Subject: [PATCH 30/35] Update tests/git.sh hashes --- tests/git.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/git.sh b/tests/git.sh index f8890ce50e2..afc39da295f 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -22,15 +22,15 @@ test "$hash1" = "sha256:1brffhvj2c0z6x8qismd43m0iy8dsgfmy10bgg9w11szway2wp9v" rm -rf $TEST_ROOT/dummy2 mkdir -p $TEST_ROOT/dummy2 -echo hello > $TEST_ROOT/dummy2/hello +echo Hello World! > $TEST_ROOT/dummy2/hello path2=$(nix add-to-store --git $TEST_ROOT/dummy2) hash2=$(nix-store -q --hash $path2) -test "$hash2" = "sha256:1pgyz59p65wd11vfxp3vi673ijwjfg7i4ynlqlsgzvg9dvh67dpj" +test "$hash2" = "sha256:1vhv7zxam7x277q0y0jcypm7hwhccbzss81vkdgf0ww5sm2am4y0" rm -rf $TEST_ROOT/dummy3 mkdir -p $TEST_ROOT/dummy3 mkdir -p $TEST_ROOT/dummy3/hello -echo hello > $TEST_ROOT/dummy3/hello/hello +echo Hello World! > $TEST_ROOT/dummy3/hello/hello path3=$(nix add-to-store --git $TEST_ROOT/dummy3) hash3=$(nix-store -q --hash $path3) -test "$hash3" = "sha256:06psnck6rb12xyd8h4y2j801xf2lvhjwx9gxzxk25h922nm71lmf" +test "$hash3" = "sha256:14qzv6n72xgr71sgb3ilpchaqzrh1vm61qj3qf0vmdfr721sm48z" From 7fe9a4891aa8c959390f87b52549ba9321bcf9fa Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 15:32:34 -0500 Subject: [PATCH 31/35] =?UTF-8?q?Don=E2=80=99t=20preallocate=20file=20size?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this shouldn’t be needed - FdSink handles this for us. --- src/libutil/git.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index f2be3eaad54..c881abb5a25 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -128,7 +128,6 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa else sink.createRegularFile(path + "/" + name); - sink.preallocateContents(st.st_size); FdSink fsink(sink.getFD()); readFile(entry, fsink); } else if (S_ISDIR(st.st_mode)) { From eb90cc65658b0bf2350e2500d547eb4847beb707 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 16:25:41 -0500 Subject: [PATCH 32/35] Copy instead of symlinking directories Symlinks are resolved in the nar format so we end up with references to the real store dir. Copying avoids this and gives us a stable hash. Also update the ParseSink api with two methods: - copyFile - copyDirectory both are needed to properly implement git parsing. --- src/libutil/fs-sink.hh | 15 ++++++++++++--- src/libutil/git.cc | 6 +++--- tests/git.sh | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 7d8764e8ba9..85e192e0743 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -1,6 +1,7 @@ #pragma once #include +#include #include "types.hh" #include "serialise.hh" @@ -20,7 +21,8 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; - virtual int getFD() { return 0; }; + virtual void copyFile(const Path & source) { }; + virtual void copyDirectory(const Path & source, const Path & destination) { }; }; struct RestoreSink : ParseSink @@ -84,9 +86,16 @@ struct RestoreSink : ParseSink nix::createSymlink(target, p); } - int getFD() + void copyFile(const Path & source) { - return fd.get(); + FdSink sink(fd.get()); + readFile(source, sink); + } + + void copyDirectory(const Path & source, const Path & destination) + { + Path p = dstPath + destination; + std::filesystem::copy(source, p); } }; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index c881abb5a25..aa96fd9bd99 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -128,13 +128,13 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa else sink.createRegularFile(path + "/" + name); - FdSink fsink(sink.getFD()); - readFile(entry, fsink); + sink.copyFile(entry); } else if (S_ISDIR(st.st_mode)) { if (perm != 40000) throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); - sink.createSymlink(path + "/" + name, "../" + entryName); + sink.createDirectory(path + "/" + name); + sink.copyDirectory(realStoreDir + "/" + entryName, path + "/" + name); } else throw Error(format("file '%1%' has an unsupported type") % entry); } } else throw Error("input doesn't look like a Git object"); diff --git a/tests/git.sh b/tests/git.sh index afc39da295f..e700dcc785c 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -33,4 +33,4 @@ mkdir -p $TEST_ROOT/dummy3/hello echo Hello World! > $TEST_ROOT/dummy3/hello/hello path3=$(nix add-to-store --git $TEST_ROOT/dummy3) hash3=$(nix-store -q --hash $path3) -test "$hash3" = "sha256:14qzv6n72xgr71sgb3ilpchaqzrh1vm61qj3qf0vmdfr721sm48z" +test "$hash3" = "sha256:1i2x80840igikhbyy7nqf08ymx3a6n83x1fzyrxvddf0sdl5nqvp" From b12025953072411dd8aedab5f0d487b504583743 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 18:29:50 -0500 Subject: [PATCH 33/35] Use custom copyDirectory instead of libc++fs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libc++fs isn’t alway available --- src/libutil/fs-sink.hh | 16 ++++++++++++++-- src/libutil/git.cc | 1 - 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 85e192e0743..efd13685df3 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -1,7 +1,6 @@ #pragma once #include -#include #include "types.hh" #include "serialise.hh" @@ -95,7 +94,20 @@ struct RestoreSink : ParseSink void copyDirectory(const Path & source, const Path & destination) { Path p = dstPath + destination; - std::filesystem::copy(source, p); + createDirectory(destination); + for (auto & i : readDirectory(source)) { + struct stat st; + Path entry = source + "/" + i.name; + if (lstat(entry.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % entry); + if (S_ISREG(st.st_mode)) { + createRegularFile(destination + "/" + i.name); + copyFile(entry); + } else if (S_ISDIR(st.st_mode)) + copyDirectory(entry, destination + "/" + i.name); + else + throw Error(format("Unknown file: %s") % entry); + } } }; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index aa96fd9bd99..3e613949a26 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -133,7 +133,6 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa if (perm != 40000) throw SysError(format("file is a directory but expected to be a file '%1%'") % entry); - sink.createDirectory(path + "/" + name); sink.copyDirectory(realStoreDir + "/" + entryName, path + "/" + name); } else throw Error(format("file '%1%' has an unsupported type") % entry); } From 7432a9dca60b963c4f8cba422bdec6967f1a1863 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 1 Jun 2020 19:06:55 -0500 Subject: [PATCH 34/35] Fixup bad rebase --- src/libstore/local-store.cc | 2 +- src/libutil/git.hh | 9 +++------ src/nix/hash.cc | 21 ++++++++------------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b742857ccbb..14c816e7282 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1132,7 +1132,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, small files. */ StringSink sink; switch (method) { - case FileIngestionMethod::Recursive: + case FileIngestionMethod::Recursive: { dumpPath(srcPath, sink, filter); break; } diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 1d773d3750f..06a56d55b70 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -25,11 +25,8 @@ typedef std::map> GitTree; GitMode dumpGitTree(const GitTree & entries, Sink & sink); // Recursively dumps path, hashing as we go -Hash dumpGitHash( - std::function()>, - const Path & path, - PathFilter & filter = defaultPathFilter); +Hash dumpGitHash(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); + +void dumpGit(HashType ht, const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); -// N.B. There is no way to recursively dump to a sink, as that doesn't make -// sense with the git hash/data model where the information is Merklized. } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 3a8c82da17b..41a66676967 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -57,34 +57,29 @@ struct CmdHash : Command { for (auto path : paths) { - auto makeHashSink = [&]() -> std::unique_ptr { - std::unique_ptr t; - if (modulus) - t = std::make_unique(ht, *modulus); - else - t = std::make_unique(ht); - return t; - }; + std::unique_ptr hashSink; + if (modulus) + hashSink = std::make_unique(ht, *modulus); + else + hashSink = std::make_unique(ht); Hash h; switch (mode) { case FileIngestionMethod::Flat: { - auto hashSink = makeHashSink(); readFile(path, *hashSink); - h = hashSink->finish().first; break; } case FileIngestionMethod::Recursive: { - auto hashSink = makeHashSink(); dumpPath(path, *hashSink); - h = hashSink->finish().first; break; } case FileIngestionMethod::Git: - h = dumpGitHash(makeHashSink, path); + dumpGit(ht, path, *hashSink); break; } + h = hashSink->finish().first; + if (truncate && h.hashSize > 20) h = compressHash(h, 20); logger->stdout(h.to_string(base, base == SRI)); } From 24544d3177324bd140d65db4068fda0b2701c6c9 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 2 Jun 2020 09:27:27 -0500 Subject: [PATCH 35/35] Verify added path matches calculated store path in add-to-store --- src/nix/add-to-store.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index ced8704ef0a..c96a16cebc8 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -61,8 +61,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(ingestionMethod, hash); - if (!dryRun) - store->addToStore(*namePart, path, ingestionMethod, git ? htSHA1 : htSHA256); + if (!dryRun) { + auto addedPath = store->addToStore(*namePart, path, ingestionMethod, git ? htSHA1 : htSHA256); + if (addedPath != info.path) + throw Error(format("Added path %s does not match calculated path %s; something has changed") % addedPath.to_string() % info.path.to_string()); + } logger->stdout("%s", store->printStorePath(info.path)); }