Trustful IPFS Store

flake.nix

@@ -88,7 +88,8 @@
             (aws-sdk-cpp.override {
               apis = ["s3" "transfer"];
               customMemoryManagement = false;
-            });
+            })
+          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ipfs;
 
         propagatedDeps =
           [ (boehmgc.override { enableLargeConfig = true; })

src/libstore/content-address.cc

@@ -1,3 +1,5 @@
+#include <nlohmann/json.hpp>
+
 #include "args.hh"
 #include "content-address.hh"
 #include "split.hh"
@@ -134,6 +136,64 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
     return ca ? renderContentAddress(*ca) : "";
 }
 
+void to_json(nlohmann::json& j, const ContentAddress & ca) {
+    j = std::visit(overloaded {
+        [](TextHash th) {
+            return nlohmann::json {
+                { "type", "text" },
+                { "hash", th.hash.to_string(Base32, false) },
+            };
+        },
+        [](FixedOutputHash foh) {
+            return nlohmann::json {
+                { "type", "fixed" },
+                { "method", foh.method == FileIngestionMethod::Flat ? "flat" : "recursive" },
+                { "algo", printHashType(foh.hash.type) },
+                { "hash", foh.hash.to_string(Base32, false) },
+            };
+        }
+    }, ca);
+}
+
+void from_json(const nlohmann::json& j, ContentAddress & ca) {
+    std::string_view type = j.at("type").get<std::string_view>();
+    if (type == "text") {
+        ca = TextHash {
+            .hash = Hash::parseNonSRIUnprefixed(j.at("hash").get<std::string_view>(), htSHA256),
+        };
+    } else if (type == "fixed") {
+        std::string_view methodRaw = j.at("method").get<std::string_view>();
+        auto method = methodRaw == "flat" ? FileIngestionMethod::Flat
+            : methodRaw == "recursive" ? FileIngestionMethod::Recursive
+            : throw Error("invalid file ingestion method: %s", methodRaw);
+        auto hashAlgo = parseHashType(j.at("algo").get<std::string>());
+        ca = FixedOutputHash {
+            .method = method,
+            .hash = Hash::parseNonSRIUnprefixed(j.at("hash").get<std::string_view>(), hashAlgo),
+        };
+    } else
+        throw Error("invalid type: %s", type);
+}
+
+// Needed until https://github.com/nlohmann/json/pull/2117
+
+void to_json(nlohmann::json& j, const std::optional<ContentAddress> & c) {
+    if (!c)
+        j = nullptr;
+    else
+        to_json(j, *c);
+}
+
+void from_json(const nlohmann::json& j, std::optional<ContentAddress> & c) {
+    if (j.is_null()) {
+        c = std::nullopt;
+    } else {
+        // Dummy value to set tag bit.
+        c = TextHash { .hash = Hash { htSHA256 } };
+        from_json(j, *c);
+    }
+}
+
 Hash getContentAddressHash(const ContentAddress & ca)
 {
     return std::visit(overloaded {

src/libstore/content-address.hh

@@ -1,6 +1,8 @@
 #pragma once
 
+#include <nlohmann/json.hpp>
 #include <variant>
+
 #include "hash.hh"
 
 namespace nix {
@@ -53,6 +55,13 @@ ContentAddress parseContentAddress(std::string_view rawCa);
 
 std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
 
+void to_json(nlohmann::json& j, const ContentAddress & c);
+void from_json(const nlohmann::json& j, ContentAddress & c);
+
+// Needed until https://github.com/nlohmann/json/pull/211
+
+void to_json(nlohmann::json& j, const std::optional<ContentAddress> & c);
+void from_json(const nlohmann::json& j, std::optional<ContentAddress> & c);
 Hash getContentAddressHash(const ContentAddress & ca);
 
 /*

src/libstore/daemon.cc

@@ -859,6 +859,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case wopSync: {
+        logger->startWork();
+        store->sync();
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
     default:
         throw Error("invalid operation %1%", op);
     }

src/libstore/filetransfer.cc

@@ -33,6 +33,8 @@ FileTransferSettings fileTransferSettings;
 
 static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
 
+MakeError(URLEncodeError, Error);
+
 std::string resolveUri(const std::string & uri)
 {
     if (uri.compare(0, 8, "channel:") == 0)
@@ -310,12 +312,22 @@ struct curlFileTransfer : public FileTransfer
 
             if (request.head)
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
+            else if (request.post)
+                curl_easy_setopt(req, CURLOPT_POST, 1);
 
             if (request.data) {
-                curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
-                curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
-                curl_easy_setopt(req, CURLOPT_READDATA, this);
-                curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
+                if (request.post) {
+                    // based off of https://curl.haxx.se/libcurl/c/postit2.html
+                    curl_mime *form = curl_mime_init(req);
+                    curl_mimepart *field = curl_mime_addpart(form);
+                    curl_mime_data(field, request.data->data(), request.data->length());
+                    curl_easy_setopt(req, CURLOPT_MIMEPOST, form);
+                } else {
+                    curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
+                    curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
+                    curl_easy_setopt(req, CURLOPT_READDATA, this);
+                    curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
+                }
             }
 
             if (request.verifyTLS) {
@@ -332,6 +344,14 @@ struct curlFileTransfer : public FileTransfer
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
 
+            /* FIXME: We hit a weird issue when 1 second goes by
+             * without Expect: 100-continue. curl_multi_perform
+             * appears to block indefinitely. To workaround this, we
+             * just set the timeout to a really big value unlikely to
+             * be hit in any server without Expect: 100-continue. This
+             * may specifically be a bug in the IPFS API. */
+            curl_easy_setopt(req, CURLOPT_EXPECT_100_TIMEOUT_MS, 300000);
+
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */
             curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
@@ -711,6 +731,22 @@ struct curlFileTransfer : public FileTransfer
 
         enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
     }
+
+    std::string urlEncode(const std::string & param) override {
+        //TODO reuse curl handle or move function to another class/file
+        CURL *curl = curl_easy_init();
+        char *encoded = NULL;
+        if (curl) {
+            encoded = curl_easy_escape(curl, param.c_str(), 0);
+        }
+        if ((curl == NULL) || (encoded == NULL)) {
+          throw URLEncodeError("Could not encode param");
+        }
+        std::string ret(encoded);
+        curl_free(encoded);
+        curl_easy_cleanup(curl);
+        return ret;
+    }
 };
 
 ref<FileTransfer> getFileTransfer()
@@ -844,6 +880,10 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
     }
 }
 
+std::string FileTransfer::urlEncode(const std::string & param) {
+    throw URLEncodeError("not implemented");
+}
+
 template<typename... Args>
 FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
     : Error(args...), error(error), response(response)

src/libstore/filetransfer.hh

@@ -55,6 +55,7 @@ struct FileTransferRequest
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
+    bool post = false;
     size_t tries = fileTransferSettings.tries;
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
@@ -105,6 +106,8 @@ struct FileTransfer
        invoked on the thread of the caller. */
     void download(FileTransferRequest && request, Sink & sink);
 
+    virtual std::string urlEncode(const std::string & param);
+
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };