diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d458ab2721f..4c72da44ad5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -725,8 +725,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { std::move(outPath), - (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") - + printHashType(h.type), + ingestionMethodPrefix(ingestionMethod) + printHashType(h.type), h.to_string(Base16, false), }); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f13736c58c0..5ca5bd3bb28 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -336,13 +336,19 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath small files. */ StringSink sink; Hash h; - if (method == FileIngestionMethod::Recursive) { + switch (method) { + case FileIngestionMethod::Recursive: dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); - } else { + break; + case FileIngestionMethod::Flat: { auto s = readFile(srcPath); dumpString(s, sink); h = hashString(hashAlgo, s); + break; + } + case FileIngestionMethod::Git: + throw Error("cannot add to binary cache store using the git file ingestion method"); } ValidPathInfo info(makeFixedOutputPath(method, h, name)); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c68e7b16b1a..acab418b63a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,13 +9,16 @@ namespace nix { -void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const +void DerivationOutput::parseHashInfo(FileIngestionMethod & method, Hash & hash) const { - recursive = FileIngestionMethod::Flat; + method = FileIngestionMethod::Flat; string algo = hashAlgo; if (string(algo, 0, 2) == "r:") { - recursive = FileIngestionMethod::Recursive; + method = FileIngestionMethod::Recursive; + algo = string(algo, 2); + } else if (string(algo, 0, 4) == "git:") { + method = FileIngestionMethod::Git; algo = string(algo, 2); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b1224b93bbb..705049d6fa4 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -22,7 +22,7 @@ struct DerivationOutput , hashAlgo(std::move(hashAlgo)) , hash(std::move(hash)) { } - void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; + void parseHashInfo(FileIngestionMethod & method, Hash & hash) const; }; typedef std::map DerivationOutputs; diff --git a/src/libstore/ipfs.cc b/src/libstore/ipfs.cc new file mode 100644 index 00000000000..3672dca2071 --- /dev/null +++ b/src/libstore/ipfs.cc @@ -0,0 +1,61 @@ +#include "binary-cache-store.hh" + +namespace nix { + +class IpfsStore : public BinaryCacheStore +{ + Path daemonUri; + +public: + + IpfsStore( + const Params & params, const Path & _daemonUri) + : BinaryCacheStore(params) + , daemonUri(_daemonUri) + { + if (daemonUri.back() == '/') + daemonUri.pop_back(); + } + + std::string getUri() override + { + return daemonUri; + } + + void init() override + { + } + + bool fileExists(const std::string & path) override + { + return false; + } + + void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) override + { + } + + void getFile(const std::string & path, Sink & sink) override + { + } + + void getFile(const std::string & path, Callback> callback) noexcept override + { + } + +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (std::string(uri, 0, 12) != "ipfs+http://") + return 0; + auto store = std::make_shared(params, std::string(uri, 5)); + store->init(); + return store; +}); + +} diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b59106..39efa650b25 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,5 +1,6 @@ #include "local-store.hh" #include "globals.hh" +#include "git.hh" #include "archive.hh" #include "pathlocks.hh" #include "worker-protocol.hh" @@ -1067,11 +1068,21 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (method == FileIngestionMethod::Recursive) { + switch (method) { + case FileIngestionMethod::Flat: + writeFile(realPath, dump); + break; + case FileIngestionMethod::Recursive: { StringSource source(dump); restorePath(realPath, source); - } else - writeFile(realPath, dump); + break; + } + case FileIngestionMethod::Git: { + StringSource source(dump); + restoreGit(realPath, source); + break; + } + } canonicalisePathMetaData(realPath, -1); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 24fab35229b..68fcf50103e 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -79,6 +79,18 @@ enum struct FileIngestionMethod : uint8_t { Git, }; +inline std::string ingestionMethodPrefix(FileIngestionMethod method) { + switch (method) { + case FileIngestionMethod::Flat: + return ""; + case FileIngestionMethod::Recursive: + return "r:"; + case FileIngestionMethod::Git: + return "git:"; + } + throw; +} + struct StorePathWithOutputs { StorePath path; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e699..4425ab9f07c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -488,6 +488,8 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + if (method == FileIngestionMethod::Git) throw Error("cannot remotely add to store using the git file ingestion method"); + auto conn(getConnection()); Path srcPath(absPath(_srcPath)); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d479b86cb11..ed8dbcbeabd 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -118,8 +118,8 @@ string storePathToHash(const Path & path) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:::", where - = "r:" for recursive (path) hashes, or "" for flat - (file) hashes + = "r:" for recursive (path) hashes, "git:" for git + paths, or "" for flat (file) hashes = "md5", "sha1" or "sha256" = base-16 representation of the path or flat hash of the contents of the path (or expected contents of the @@ -172,20 +172,20 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - FileIngestionMethod recursive, + FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { + if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + ingestionMethodPrefix(method) + hash.to_string(Base16) + ":"), name); } @@ -787,15 +787,19 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const } else if (hasPrefix(ca, "fixed:")) { - FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6)); + FileIngestionMethod method = FileIngestionMethod::Flat; + if (ca.compare(6, 2, "r:") == 0) + method = FileIngestionMethod::Recursive; + else if (ca.compare(6, 4, "git:") == 0) + method = FileIngestionMethod::Git; + Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length())); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; refs.erase(path); } - if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path) + if (store.makeFixedOutputPath(method, hash, path.name(), refs, hasSelfReference) == path) return true; else warn(); @@ -832,11 +836,9 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) { - return "fixed:" - + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") - + hash.to_string(); + return "fixed:" + ingestionMethodPrefix(method) + hash.to_string(); } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index db544a212ab..3670131c291 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -294,61 +294,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError(format("creating directory '%1%'") % p); - }; - - void createRegularFile(const Path & path) - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError(format("creating file '%1%'") % p); - } - - void isExecutable() - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(unsigned long long len) - { -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError(format("preallocating file of %1% bytes") % len); - } -#endif - } - - void receiveContents(unsigned char * data, unsigned int len) - { - writeFull(fd.get(), data, len); - } - - void createSymlink(const Path & path, const string & target) - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - void restorePath(const Path & path, Source & source) { diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 768fe25368d..14bf9740d91 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -2,7 +2,7 @@ #include "types.hh" #include "serialise.hh" - +#include "fs-sink.hh" namespace nix { @@ -50,19 +50,6 @@ void dumpPath(const Path & path, Sink & sink, void dumpString(const std::string & s, Sink & sink); -/* FIXME: fix this API, it sucks. */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void isExecutable() { }; - virtual void preallocateContents(unsigned long long size) { }; - virtual void receiveContents(unsigned char * data, unsigned int len) { }; - - virtual void createSymlink(const Path & path, const string & target) { }; -}; - struct TeeSink : ParseSink { TeeSource source; diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh new file mode 100644 index 00000000000..d3d1e43b79f --- /dev/null +++ b/src/libutil/fs-sink.hh @@ -0,0 +1,79 @@ +#pragma once + +#include + +#include "types.hh" +#include "serialise.hh" + +namespace nix { + +/* FIXME: fix this API, it sucks. */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void isExecutable() { }; + virtual void preallocateContents(unsigned long long size) { }; + virtual void receiveContents(unsigned char * data, unsigned int len) { }; + + virtual void createSymlink(const Path & path, const string & target) { }; +}; + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError(format("creating directory '%1%'") % p); + }; + + void createRegularFile(const Path & path) + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError(format("creating file '%1%'") % p); + } + + void isExecutable() + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(unsigned long long len) + { +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError(format("preallocating file of %1% bytes") % len); + } +#endif + } + + void receiveContents(unsigned char * data, unsigned int len) + { + writeFull(fd.get(), data, len); + } + + void createSymlink(const Path & path, const string & target) + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + + +} diff --git a/src/libutil/git.cc b/src/libutil/git.cc index bac98d1bfd8..1f44aa1e1f2 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -1,12 +1,56 @@ -#include "archive.hh" -#include "hash.hh" +#include +#include +#include +#include + +#include // for strcasecmp + +#include +#include +#include +#include +#include + #include "util.hh" #include "config.hh" +#include "hash.hh" + +#include "git.hh" using namespace std::string_literals; namespace nix { +static void parse(ParseSink & sink, Source & source, const Path & path); + +// Converts a Path to a ParseSink +void restoreGit(const Path & path, Source & source) { + + RestoreSink sink; + sink.dstPath = path; + parseGit(sink, source); + +} + +void parseGit(ParseSink & sink, Source & source) { + parse(sink, source, ""); +} + +static void parse(ParseSink & sink, Source & source, const Path & path) { + uint8_t buf[4]; + + std::basic_string_view buf_v { + (const uint8_t *) & buf, + std::size(buf) + }; + source(buf_v); + if (buf_v.compare((const uint8_t *)"blob")) { + uint8_t space; + source(& space, 1); + } + else { + } +} // Internal version, returns the perm. unsigned int dumpGitInternal(const Path & path, Sink & sink, PathFilter & filter) { diff --git a/src/libutil/git.hh b/src/libutil/git.hh index e2d978aafae..f87df52d8cc 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -2,9 +2,20 @@ #include "types.hh" #include "serialise.hh" +#include "fs-sink.hh" namespace nix { +enum struct GitMode { + Directory, + Executable, + Regular, +}; + +void restoreGit(const Path & path, Source & source); + +void parseGit(ParseSink & sink, Source & source); + void dumpGit(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8201549fd7d..79b6048e6ee 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -89,6 +89,10 @@ void Source::operator () (unsigned char * data, size_t len) } } +void Source::operator () (std::basic_string_view & data) +{ + (*this)((unsigned char *) data.data(), data.size()); +} std::string Source::drain() { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a04118512fc..7bcd476aafc 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -54,6 +54,7 @@ struct Source It blocks until all the requested data is available, or throws an error if it is not going to be available. */ void operator () (unsigned char * data, size_t len); + void operator () (std::basic_string_view & data); /* Store up to ‘len’ in the buffer pointed to by ‘data’, and return the number of bytes stored. It blocks until at least diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3a3060ad82d..68fe6f72735 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.empty()) @@ -187,17 +187,17 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo))); + cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, method, hashAlgo))); } /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.size() != 3) @@ -208,7 +208,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(method, Hash(hash, hashAlgo), name))); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 3691ea73880..db3f00d677a 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -54,25 +54,36 @@ struct CmdHash : Command { for (auto path : paths) { - std::unique_ptr hashSink; - if (modulus) - hashSink = std::make_unique(ht, *modulus); - else - hashSink = std::make_unique(ht); - + auto makeHashSink = [&]() -> std::unique_ptr { + std::unique_ptr t; + if (modulus) + t = std::make_unique(ht, *modulus); + else + t = std::make_unique(ht); + return t; + }; + + Hash h; switch (mode) { - case FileIngestionMethod::Flat: + case FileIngestionMethod::Flat: { + auto hashSink = makeHashSink(); readFile(path, *hashSink); + h = hashSink->finish().first; break; - case FileIngestionMethod::Recursive: + } + case FileIngestionMethod::Recursive: { + auto hashSink = makeHashSink(); dumpPath(path, *hashSink); + h = hashSink->finish().first; break; + } case FileIngestionMethod::Git: + auto hashSink = makeHashSink(); dumpGit(path, *hashSink); + h = hashSink->finish().first; break; } - Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); logger->stdout(h.to_string(base, base == SRI)); }