Merge pull request #3 from obsidiansystems/ipfs-git-addtostore

Add git add-to-store support for Nix

src/libstore/daemon.cc

@@ -8,6 +8,7 @@
 #include "archive.hh"
 #include "derivations.hh"
 #include "args.hh"
+#include "git.hh"
 
 namespace nix::daemon {
 
@@ -358,7 +359,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         std::string s, baseName;
         FileIngestionMethod method;
         {
-            bool fixed, recursive;
+            bool fixed;
+            unsigned char recursive;
             from >> baseName >> fixed /* obsolete */ >> recursive >> s;
             method = FileIngestionMethod { recursive };
             /* Compatibility hack. */
@@ -372,20 +374,31 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         TeeSource savedNAR(from);
         RetrieveRegularNARSink savedRegular;
 
-        if (method == FileIngestionMethod::Recursive) {
+        switch (method) {
+        case FileIngestionMethod::Recursive: {
             /* Get the entire NAR dump from the client and save it to
                a string so that we can pass it to
                addToStoreFromDump(). */
             ParseSink sink; /* null sink; just parse the NAR */
             parseDump(sink, savedNAR);
-        } else
+            break;
+        }
+        case FileIngestionMethod::Git: {
+            ParseSink sink;
+            parseGit(sink, savedNAR, store->storeDir, store->storeDir);
+            break;
+        }
+        case FileIngestionMethod::Flat: {
             parseDump(savedRegular, from);
+            break;
+        }
+        }
 
         logger->startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
 
         auto path = store->addToStoreFromDump(
-            method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
+            method == FileIngestionMethod::Flat ? savedRegular.s : *savedNAR.data,
             baseName,
             method,
             hashAlgo);

src/libstore/local-store.cc

@@ -1017,7 +1017,10 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
                 return n;
             });
 
-            restorePath(realPath, wrapperSource);
+            if (hasPrefix(info.ca, "fixed:git:"))
+                restoreGit(realPath, wrapperSource, realStoreDir, storeDir);
+            else
+                restorePath(realPath, wrapperSource);
 
             auto hashResult = hashSink->finish();
 
@@ -1046,6 +1049,9 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
     FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
 {
+    if (method == FileIngestionMethod::Git && hashAlgo != htSHA1)
+        throw Error("git ingestion must use sha1 hash");
+
     Hash h = hashString(hashAlgo, dump);
 
     auto dstPath = makeFixedOutputPath(method, h, name);
@@ -1079,7 +1085,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
             }
             case FileIngestionMethod::Git: {
                 StringSource source(dump);
-                restoreGit(realPath, source);
+                restoreGit(realPath, source, realStoreDir, storeDir);
                 break;
             }
             }
@@ -1118,14 +1124,35 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
 {
     Path srcPath(absPath(_srcPath));
 
+    if (method == FileIngestionMethod::Git && hashAlgo != htSHA1)
+        throw Error("git ingestion must use sha1 hash");
+
     /* Read the whole path into memory. This is not a very scalable
        method for very large paths, but `copyPath' is mainly used for
        small files. */
     StringSink sink;
-    if (method == FileIngestionMethod::Recursive)
+    switch (method) {
+    case FileIngestionMethod::Recursive: {
         dumpPath(srcPath, sink, filter);
-    else
+        break;
+    }
+    case FileIngestionMethod::Git: {
+        // recursively add to store if path is a directory
+        struct stat st;
+        if (lstat(srcPath.c_str(), &st))
+            throw SysError(format("getting attributes of path '%1%'") % srcPath);
+        if (S_ISDIR(st.st_mode))
+            for (auto & i : readDirectory(srcPath))
+                addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair);
+
+        dumpGit(hashAlgo, srcPath, sink, filter);
+        break;
+    }
+    case FileIngestionMethod::Flat: {
         sink.s = make_ref<std::string>(readFile(srcPath));
+        break;
+    }
+    }
 
     return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
 }

src/libstore/remote-store.cc

@@ -8,6 +8,7 @@
 #include "derivations.hh"
 #include "pool.hh"
 #include "finally.hh"
+#include "git.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -488,17 +489,28 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
-    if (method == FileIngestionMethod::Git) throw Error("cannot remotely add to store using the git file ingestion method");
-
-    auto conn(getConnection());
+    if (method == FileIngestionMethod::Git && hashAlgo != htSHA1)
+        throw Error("git ingestion must use sha1 hash");
 
     Path srcPath(absPath(_srcPath));
 
+    // recursively add to store if path is a directory
+    if (method == FileIngestionMethod::Git) {
+        struct stat st;
+        if (lstat(srcPath.c_str(), &st))
+            throw SysError(format("getting attributes of path '%1%'") % srcPath);
+        if (S_ISDIR(st.st_mode))
+            for (auto & i : readDirectory(srcPath))
+                addToStore(i.name, srcPath + "/" + i.name, method, hashAlgo, filter, repair);
+    }
+
+    auto conn(getConnection());
+
     conn->to
         << wopAddToStore
         << name
         << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
-        << (method == FileIngestionMethod::Recursive ? 1 : 0)
+        << (uint8_t) method
         << printHashType(hashAlgo);
 
     try {
@@ -507,7 +519,10 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
         connections->incCapacity();
         {
             Finally cleanup([&]() { connections->decCapacity(); });
-            dumpPath(srcPath, conn->to, filter);
+            if (method == FileIngestionMethod::Git)
+                dumpGit(hashAlgo, srcPath, conn->to, filter);
+            else
+                dumpPath(srcPath, conn->to, filter);
         }
         conn->to.warn = false;
         conn.processStderr();

src/libstore/store-api.cc

@@ -178,6 +178,9 @@ StorePath Store::makeFixedOutputPath(
     const StorePathSet & references,
     bool hasSelfReference) const
 {
+    if (method == FileIngestionMethod::Git && hash.type != htSHA1)
+        throw Error("Git file ingestion must use sha1 hash");
+
     if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
     } else {
@@ -206,9 +209,21 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
 std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
     const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const
 {
-    Hash h = method == FileIngestionMethod::Recursive
-        ? hashPath(hashAlgo, srcPath, filter).first
-        : hashFile(hashAlgo, srcPath);
+    Hash h;
+    switch (method) {
+    case FileIngestionMethod::Recursive: {
+        h = hashPath(hashAlgo, srcPath, filter).first;
+        break;
+    }
+    case FileIngestionMethod::Git: {
+        h = hashGit(hashAlgo, srcPath, filter).first;
+        break;
+    }
+    case FileIngestionMethod::Flat: {
+        h = hashFile(hashAlgo, srcPath);
+        break;
+    }
+    }
     return std::make_pair(makeFixedOutputPath(method, h, name), h);
 }
 
@@ -838,6 +853,8 @@ Strings ValidPathInfo::shortRefs() const
 
 std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
 {
+    if (method == FileIngestionMethod::Git && hash.type != htSHA1)
+        throw Error("git file ingestion must use sha1 hashes");
     return "fixed:" + ingestionMethodPrefix(method) + hash.to_string();
 }
 

src/libutil/fs-sink.hh

@@ -13,11 +13,15 @@ struct ParseSink
     virtual void createDirectory(const Path & path) { };
 
     virtual void createRegularFile(const Path & path) { };
+    virtual void createExecutableFile(const Path & path) { };
     virtual void isExecutable() { };
     virtual void preallocateContents(unsigned long long size) { };
     virtual void receiveContents(unsigned char * data, unsigned int len) { };
 
     virtual void createSymlink(const Path & path, const string & target) { };
+
+    virtual void copyFile(const Path & source) { };
+    virtual void copyDirectory(const Path & source, const Path & destination) { };
 };
 
 struct RestoreSink : ParseSink
@@ -39,6 +43,13 @@ struct RestoreSink : ParseSink
         if (!fd) throw SysError(format("creating file '%1%'") % p);
     }
 
+    void createExecutableFile(const Path & path)
+    {
+        Path p = dstPath + path;
+        fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0777);
+        if (!fd) throw SysError(format("creating file '%1%'") % p);
+    }
+
     void isExecutable()
     {
         struct stat st;
@@ -73,6 +84,31 @@ struct RestoreSink : ParseSink
         Path p = dstPath + path;
         nix::createSymlink(target, p);
     }
+
+    void copyFile(const Path & source)
+    {
+        FdSink sink(fd.get());
+        readFile(source, sink);
+    }
+
+    void copyDirectory(const Path & source, const Path & destination)
+    {
+        Path p = dstPath + destination;
+        createDirectory(destination);
+        for (auto & i : readDirectory(source)) {
+            struct stat st;
+            Path entry = source + "/" + i.name;
+            if (lstat(entry.c_str(), &st))
+                throw SysError(format("getting attributes of path '%1%'") % entry);
+            if (S_ISREG(st.st_mode)) {
+                createRegularFile(destination + "/" + i.name);
+                copyFile(entry);
+            } else if (S_ISDIR(st.st_mode))
+                copyDirectory(entry, destination + "/" + i.name);
+            else
+                throw Error(format("Unknown file: %s") % entry);
+        }
+    }
 };