Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Proper parse and render functions for FileIngestionMethod and ContentAddressMethod #10009

Merged
merged 3 commits into from
Feb 13, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 2 additions & 16 deletions src/libfetchers/fetch-to-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -21,23 +21,9 @@ StorePath fetchToStore(
cacheKey = fetchers::Attrs{
{"_what", "fetchToStore"},
{"store", store.storeDir},
{"name", std::string(name)},
{"name", std::string{name}},
{"fingerprint", *path.accessor->fingerprint},
{
"method",
std::visit(overloaded {
[](const TextIngestionMethod &) {
return "text";
},
[](const FileIngestionMethod & fim) {
switch (fim) {
case FileIngestionMethod::Flat: return "flat";
case FileIngestionMethod::Recursive: return "nar";
default: assert(false);
}
},
}, method.raw),
},
{"method", std::string{method.render()}},
{"path", path.path.abs()}
};
if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
Expand Down
31 changes: 25 additions & 6 deletions src/libstore/content-address.cc
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

namespace nix {

std::string makeFileIngestionPrefix(FileIngestionMethod m)
std::string_view makeFileIngestionPrefix(FileIngestionMethod m)
{
switch (m) {
case FileIngestionMethod::Flat:
Expand All @@ -16,10 +16,29 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m)
}
}

std::string ContentAddressMethod::renderPrefix() const
std::string_view ContentAddressMethod::render() const
{
return std::visit(overloaded {
[](TextIngestionMethod) -> std::string { return "text:"; },
[](TextIngestionMethod) -> std::string_view { return "text"; },
[](FileIngestionMethod m2) {
/* Not prefixed for back compat with things that couldn't produce text before. */
return renderFileIngestionMethod(m2);
},
}, raw);
}

ContentAddressMethod ContentAddressMethod::parse(std::string_view m)
{
if (m == "text")
return TextIngestionMethod {};
else
return parseFileIngestionMethod(m);
}

std::string_view ContentAddressMethod::renderPrefix() const
{
return std::visit(overloaded {
[](TextIngestionMethod) -> std::string_view { return "text:"; },
[](FileIngestionMethod m2) {
/* Not prefixed for back compat with things that couldn't produce text before. */
return makeFileIngestionPrefix(m2);
Expand All @@ -38,7 +57,7 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
return FileIngestionMethod::Flat;
}

std::string ContentAddressMethod::render(HashAlgorithm ha) const
std::string ContentAddressMethod::renderWithAlgo(HashAlgorithm ha) const
{
return std::visit(overloaded {
[&](const TextIngestionMethod & th) {
Expand Down Expand Up @@ -133,7 +152,7 @@ ContentAddress ContentAddress::parse(std::string_view rawCa)
};
}

std::pair<ContentAddressMethod, HashAlgorithm> ContentAddressMethod::parse(std::string_view caMethod)
std::pair<ContentAddressMethod, HashAlgorithm> ContentAddressMethod::parseWithAlgo(std::string_view caMethod)
{
std::string asPrefix = std::string{caMethod} + ":";
// parseContentAddressMethodPrefix takes its argument by reference
Expand All @@ -155,7 +174,7 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)

std::string ContentAddress::printMethodAlgo() const
{
return method.renderPrefix()
return std::string { method.renderPrefix() }
+ printHashAlgo(hash.algo);
}

Expand Down
22 changes: 18 additions & 4 deletions src/libstore/content-address.hh
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ struct TextIngestionMethod : std::monostate { };
* Compute the prefix to the hash algorithm which indicates how the
* files were ingested.
*/
std::string makeFileIngestionPrefix(FileIngestionMethod m);
std::string_view makeFileIngestionPrefix(FileIngestionMethod m);

/**
* An enumeration of all the ways we can content-address store objects.
Expand All @@ -59,6 +59,20 @@ struct ContentAddressMethod

MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod);

/**
* Parse a content addressing method (name).
*
* The inverse of `render`.
*/
static ContentAddressMethod parse(std::string_view rawCaMethod);

/**
* Render a content addressing method (name).
*
* The inverse of `parse`.
*/
std::string_view render() const;

/**
* Parse the prefix tag which indicates how the files
* were ingested, with the fixed output case not prefixed for back
Expand All @@ -74,20 +88,20 @@ struct ContentAddressMethod
*
* The rough inverse of `parsePrefix()`.
*/
std::string renderPrefix() const;
std::string_view renderPrefix() const;

/**
* Parse a content addressing method and hash type.
*/
static std::pair<ContentAddressMethod, HashAlgorithm> parse(std::string_view rawCaMethod);
static std::pair<ContentAddressMethod, HashAlgorithm> parseWithAlgo(std::string_view rawCaMethod);

/**
* Render a content addressing method and hash type in a
* nicer way, prefixing both cases.
*
* The rough inverse of `parse()`.
*/
std::string render(HashAlgorithm ht) const;
std::string renderWithAlgo(HashAlgorithm ht) const;

/**
* Get the underlying way to content-address file system objects.
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/daemon.cc
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto pathInfo = [&]() {
// NB: FramedSource must be out of scope before logger->stopWork();
auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr);
auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parseWithAlgo(camStr);
auto hashAlgo = hashAlgo_; // work around clang bug
FramedSource source(from);
// TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.
Expand Down
12 changes: 6 additions & 6 deletions src/libstore/derivations.cc
Original file line number Diff line number Diff line change
Expand Up @@ -601,7 +601,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
},
[&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo));
s += ','; printUnquotedString(s, std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo));
s += ','; printUnquotedString(s, "");
},
[&](const DerivationOutput::Deferred &) {
Expand All @@ -612,7 +612,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
[&](const DerivationOutput::Impure & doi) {
// FIXME
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo));
s += ','; printUnquotedString(s, std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo));
s += ','; printUnquotedString(s, "impure");
}
}, i.second.raw);
Expand Down Expand Up @@ -984,7 +984,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
},
[&](const DerivationOutput::CAFloating & dof) {
out << ""
<< (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo))
<< (std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo))
<< "";
},
[&](const DerivationOutput::Deferred &) {
Expand All @@ -994,7 +994,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
},
[&](const DerivationOutput::Impure & doi) {
out << ""
<< (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo))
<< (std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo))
<< "impure";
},
}, i.second.raw);
Expand Down Expand Up @@ -1221,11 +1221,11 @@ nlohmann::json DerivationOutput::toJSON(
// FIXME print refs?
},
[&](const DerivationOutput::CAFloating & dof) {
res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo);
res["hashAlgo"] = std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo);
},
[&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) {
res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo);
res["hashAlgo"] = std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo);
res["impure"] = true;
},
}, raw);
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/remote-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -435,7 +435,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
conn->to
<< WorkerProto::Op::AddToStore
<< name
<< caMethod.render(hashAlgo);
<< caMethod.renderWithAlgo(hashAlgo);
WorkerProto::write(*this, *conn, references);
conn->to << repair;

Expand Down
25 changes: 25 additions & 0 deletions src/libutil/file-content-address.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,31 @@

namespace nix {

FileIngestionMethod parseFileIngestionMethod(std::string_view input)
{
if (input == "flat") {
return FileIngestionMethod::Flat;
} else if (input == "nar") {
return FileIngestionMethod::Recursive;
} else {
throw UsageError("Unknown file ingestion method '%s', expect `flat` or `nar`");
}
}


std::string_view renderFileIngestionMethod(FileIngestionMethod method)
{
switch (method) {
case FileIngestionMethod::Flat:
return "flat";
case FileIngestionMethod::Recursive:
return "nar";
default:
assert(false);
Ericson2314 marked this conversation as resolved.
Show resolved Hide resolved
}
}


void dumpPath(
SourceAccessor & accessor, const CanonPath & path,
Sink & sink,
Expand Down
17 changes: 17 additions & 0 deletions src/libutil/file-content-address.hh
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,23 @@ enum struct FileIngestionMethod : uint8_t {
Recursive = 1,
};

/**
* Parse a `FileIngestionMethod` by name. Choice of:
*
* - `flat`: `FileIngestionMethod::Flat`
* - `nar`: `FileIngestionMethod::Recursive`
*
* Oppostite of `renderFileIngestionMethod`.
*/
FileIngestionMethod parseFileIngestionMethod(std::string_view input);

/**
* Render a `FileIngestionMethod` by name.
*
* Oppostite of `parseFileIngestionMethod`.
*/
std::string_view renderFileIngestionMethod(FileIngestionMethod method);

/**
* Dump a serialization of the given file system object.
*/
Expand Down
13 changes: 1 addition & 12 deletions src/nix/add-to-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,6 @@

using namespace nix;

static FileIngestionMethod parseIngestionMethod(std::string_view input)
{
if (input == "flat") {
return FileIngestionMethod::Flat;
} else if (input == "nar") {
return FileIngestionMethod::Recursive;
} else {
throw UsageError("Unknown hash mode '%s', expect `flat` or `nar`");
}
}

struct CmdAddToStore : MixDryRun, StoreCommand
{
Path path;
Expand Down Expand Up @@ -49,7 +38,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
)",
.labels = {"hash-mode"},
.handler = {[this](std::string s) {
this->caMethod = parseIngestionMethod(s);
this->caMethod = parseFileIngestionMethod(s);
}},
});

Expand Down
35 changes: 35 additions & 0 deletions tests/unit/libstore/content-address.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#include <gtest/gtest.h>

#include "content-address.hh"

namespace nix {

/* ----------------------------------------------------------------------------
* ContentAddressMethod::parse, ContentAddressMethod::render
* --------------------------------------------------------------------------*/

TEST(ContentAddressMethod, testRoundTripPrintParse_1) {
for (const ContentAddressMethod & cam : {
ContentAddressMethod { TextIngestionMethod {} },
ContentAddressMethod { FileIngestionMethod::Flat },
ContentAddressMethod { FileIngestionMethod::Recursive },
}) {
EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
}
}

TEST(ContentAddressMethod, testRoundTripPrintParse_2) {
for (const std::string_view camS : {
"text",
"flat",
"nar",
}) {
EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
}
}

TEST(ContentAddressMethod, testParseContentAddressMethodOptException) {
EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError);
}

}
33 changes: 33 additions & 0 deletions tests/unit/libutil/file-content-address.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#include <gtest/gtest.h>

#include "file-content-address.hh"

namespace nix {

/* ----------------------------------------------------------------------------
* parseFileIngestionMethod, renderFileIngestionMethod
* --------------------------------------------------------------------------*/

TEST(FileIngestionMethod, testRoundTripPrintParse_1) {
for (const FileIngestionMethod fim : {
FileIngestionMethod::Flat,
FileIngestionMethod::Recursive,
}) {
EXPECT_EQ(parseFileIngestionMethod(renderFileIngestionMethod(fim)), fim);
}
}

TEST(FileIngestionMethod, testRoundTripPrintParse_2) {
for (const std::string_view fimS : {
"flat",
"nar",
}) {
EXPECT_EQ(renderFileIngestionMethod(parseFileIngestionMethod(fimS)), fimS);
}
}

TEST(FileIngestionMethod, testParseFileIngestionMethodOptException) {
EXPECT_THROW(parseFileIngestionMethod("narwhal"), UsageError);
}

}
Loading