Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hash always has a valid type #3738

Merged
merged 16 commits into from
Jul 27, 2020
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions perl/lib/Nix/Store.xs
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ SV * queryReferences(char * path)
SV * queryPathHash(char * path)
PPCODE:
try {
auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
Expand All @@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
XPUSHs(&PL_sv_undef);
else
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
auto s = info->narHash->to_string(base32 ? Base32 : Base16, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize);
Expand Down
8 changes: 4 additions & 4 deletions src/libexpr/primops.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1122,7 +1122,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu


static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v)
Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
{
const auto path = evalSettings.pureEval && expectedHash ?
path_ :
Expand Down Expand Up @@ -1153,7 +1153,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con

std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name);
expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode
Expand Down Expand Up @@ -1187,7 +1187,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
.nixCode = NixCode { .errPos = pos }
});

addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
}

static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
Expand All @@ -1197,7 +1197,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
string name;
Value * filterFun = nullptr;
auto method = FileIngestionMethod::Recursive;
Hash expectedHash;
Hash expectedHash(htSHA256);

for (auto & attr : *args[0]->attrs) {
const string & n(attr.name);
Expand Down
4 changes: 2 additions & 2 deletions src/libexpr/primops/fetchTree.cc
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ void emitTreeAttrs(

assert(tree.info.narHash);
mkString(*state.allocAttr(v, state.symbols.create("narHash")),
tree.info.narHash.to_string(SRI, true));
tree.info.narHash->to_string(SRI, true));

if (input->getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
Expand Down Expand Up @@ -147,7 +147,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
: hashFile(htSHA256, path);
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
*url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true));
}

if (state.allowedPaths)
Expand Down
2 changes: 1 addition & 1 deletion src/libfetchers/tree-info.cc
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ namespace nix::fetchers {
StorePath TreeInfo::computeStorePath(Store & store) const
{
assert(narHash);
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
}

}
2 changes: 1 addition & 1 deletion src/libfetchers/tree-info.hh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ namespace nix::fetchers {

struct TreeInfo
{
Hash narHash;
std::optional<Hash> narHash;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Though I think this one is fine and just means the hash is calculted---obviously these builtins aren't going to be making non-CA paths.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this one should not be optional. Anyway it's better to leave libfetchers unchanged since it's all rewritten in the flakes branch.

Copy link
Member Author

@Ericson2314 Ericson2314 Jul 7, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We thought it might be easy to fix this, but it isn't either: the caching stuff doesn't store the narHash and since the cache also doesn't distinguish between stores, doing a queryPathInfo to get the Nar hash in the case of a cache-hit doesn't work reliably enough to pass the test suite.

Perhaps something could be made to work, but like you said libfetchers is all rewritten anyways, and I think the 2 lines we changed in libfetchers with the std::optional is much more minimal than whatever it would take to make it work without the std::optional.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change did go away with the flakes branch being merged, yay!

std::optional<uint64_t> revCount;
std::optional<time_t> lastModified;

Expand Down
6 changes: 3 additions & 3 deletions src/libstore/binary-cache-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
duration);

narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
Expand Down Expand Up @@ -338,7 +338,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
Hash h;
std::optional<Hash> h;
if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s);
Expand All @@ -348,7 +348,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
h = hashString(hashAlgo, s);
}

ValidPathInfo info(makeFixedOutputPath(method, h, name));
ValidPathInfo info(makeFixedOutputPath(method, *h, name));

auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
Expand Down
12 changes: 7 additions & 5 deletions src/libstore/build.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3733,8 +3733,8 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
? hashPath(*i.second.hash->hash.type, actualPath).first
: hashFile(*i.second.hash->hash.type, actualPath);
? hashPath(i.second.hash->hash.type, actualPath).first
: hashFile(i.second.hash->hash.type, actualPath);

auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());

Expand Down Expand Up @@ -3783,8 +3783,10 @@ void DerivationGoal::registerOutputs()
time. The hash is stored in the database so that we can
verify later on whether nobody has messed with the store. */
debug("scanning for references inside '%1%'", path);
HashResult hash;
auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash));
// HashResult hash;
auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths));
auto references = worker.store.parseStorePathSet(pathSetAndHash.first);
HashResult hash = pathSetAndHash.second;

if (buildMode == bmCheck) {
if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue;
Expand Down Expand Up @@ -5006,7 +5008,7 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
HashResult current = hashPath(*info->narHash.type, store.printStorePath(path));
HashResult current = hashPath(info->narHash->type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/builtins/fetchurl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
auto & h = output.hash->hash;
fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
return;
} catch (Error & e) {
debug(e.what());
Expand Down
4 changes: 2 additions & 2 deletions src/libstore/content-address.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
namespace nix {

std::string FixedOutputHash::printMethodAlgo() const {
return makeFileIngestionPrefix(method) + printHashType(*hash.type);
return makeFileIngestionPrefix(method) + printHashType(hash.type);
}

std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
Expand Down Expand Up @@ -46,7 +46,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
if (prefix == "text") {
auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
Hash hash = Hash(string(hashTypeAndHash));
if (*hash.type != htSHA256) {
if (hash.type != htSHA256) {
throw Error("parseContentAddress: the text hash should have type SHA256");
}
return TextHash { hash };
Expand Down
4 changes: 2 additions & 2 deletions src/libstore/daemon.cc
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto hash = store->queryPathInfo(path)->narHash;
logger->stopWork();
to << hash.to_string(Base16, false);
to << hash->to_string(Base16, false);
break;
}

Expand Down Expand Up @@ -655,7 +655,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1;
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
<< info->narHash.to_string(Base16, false);
<< info->narHash->to_string(Base16, false);
writeStorePaths(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
Expand Down
4 changes: 2 additions & 2 deletions src/libstore/export-import.cc
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ void Store::exportPath(const StorePath & path, Sink & sink)
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
if (hash != info->narHash && info->narHash != Hash(info->narHash->type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true));

hashAndWriteSink
<< exportMagic
Expand Down
4 changes: 2 additions & 2 deletions src/libstore/legacy-ssh-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store

if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
info->narHash = s.empty() ? Hash() : Hash(s);
info->narHash = s.empty() ? std::optional<Hash>{} : Hash{s};
info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from);
}
Expand All @@ -139,7 +139,7 @@ struct LegacySSHStore : public Store
<< cmdAddToStoreNar
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
<< info.narHash->to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to
<< info.registrationTime
Expand Down
26 changes: 13 additions & 13 deletions src/libstore/local-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -586,7 +586,7 @@ uint64_t LocalStore::addValidPath(State & state,

state.stmtRegisterValidPath.use()
(printStorePath(info.path))
(info.narHash.to_string(Base16, true))
(info.narHash->to_string(Base16, true))
(info.registrationTime == 0 ? time(0) : info.registrationTime)
(info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
(info.narSize, info.narSize != 0)
Expand Down Expand Up @@ -686,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{
state.stmtUpdatePathInfo.use()
(info.narSize, info.narSize != 0)
(info.narHash.to_string(Base16, true))
(info.narHash->to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(renderContentAddress(info.ca), (bool) info.ca)
Expand Down Expand Up @@ -900,7 +900,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
StorePathSet paths;

for (auto & i : infos) {
assert(i.narHash.type == htSHA256);
assert(i.narHash && i.narHash->type == htSHA256);
if (isValidPath_(*state, i.path))
updatePathInfo(*state, i);
else
Expand Down Expand Up @@ -1013,7 +1013,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,

if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true));

if (hashResult.second != info.narSize)
throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s",
Expand Down Expand Up @@ -1070,12 +1070,12 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
the path in the database. We may just have computed it
above (if called with recursive == true and hashAlgo ==
sha256); otherwise, compute it here. */
HashResult hash;
if (method == FileIngestionMethod::Recursive) {
hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
hash.second = dump.size();
} else
hash = hashPath(htSHA256, realPath);
HashResult hash = method == FileIngestionMethod::Recursive
? HashResult {
hashAlgo == htSHA256 ? h : hashString(htSHA256, dump),
dump.size(),
}
: hashPath(htSHA256, realPath);

optimisePath(realPath); // FIXME: combine with hashPath()

Expand Down Expand Up @@ -1258,9 +1258,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)

std::unique_ptr<AbstractHashSink> hashSink;
if (!info->ca || !info->references.count(info->path))
hashSink = std::make_unique<HashSink>(*info->narHash.type);
hashSink = std::make_unique<HashSink>(info->narHash->type);
else
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
hashSink = std::make_unique<HashModuloSink>(info->narHash->type, std::string(info->path.hashPart()));

dumpPath(Store::toRealPath(i), *hashSink);
auto current = hashSink->finish();
Expand All @@ -1269,7 +1269,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
logError({
.name = "Invalid hash - path modified",
.hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true))
});
if (repair) repairPath(i); else errors = true;
} else {
Expand Down
4 changes: 2 additions & 2 deletions src/libstore/nar-info-disk-cache.cc
Original file line number Diff line number Diff line change
Expand Up @@ -230,9 +230,9 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
(std::string(info->path.name()))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash)
(narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash)
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
(info->narHash.to_string(Base32, true))
(info->narHash->to_string(Base32, true))
(info->narSize)
(concatStringsSep(" ", info->shortRefs()))
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
Expand Down
27 changes: 13 additions & 14 deletions src/libstore/nar-info.cc
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,14 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
: ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack
{
auto corrupt = [&]() {
throw Error("NAR info file '%1%' is corrupt", whence);
return Error("NAR info file '%1%' is corrupt", whence);
};

auto parseHashField = [&](const string & s) {
try {
return Hash(s);
} catch (BadHash &) {
corrupt();
return Hash(); // never reached
throw corrupt();
}
};

Expand All @@ -25,12 +24,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
while (pos < s.size()) {

size_t colon = s.find(':', pos);
if (colon == std::string::npos) corrupt();
if (colon == std::string::npos) throw corrupt();

std::string name(s, pos, colon - pos);

size_t eol = s.find('\n', colon + 2);
if (eol == std::string::npos) corrupt();
if (eol == std::string::npos) throw corrupt();

std::string value(s, colon + 2, eol - colon - 2);

Expand All @@ -45,16 +44,16 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "FileHash")
fileHash = parseHashField(value);
else if (name == "FileSize") {
if (!string2Int(value, fileSize)) corrupt();
if (!string2Int(value, fileSize)) throw corrupt();
}
else if (name == "NarHash")
narHash = parseHashField(value);
else if (name == "NarSize") {
if (!string2Int(value, narSize)) corrupt();
if (!string2Int(value, narSize)) throw corrupt();
}
else if (name == "References") {
auto refs = tokenizeString<Strings>(value, " ");
if (!references.empty()) corrupt();
if (!references.empty()) throw corrupt();
for (auto & r : refs)
references.insert(StorePath(r));
}
Expand All @@ -67,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "Sig")
sigs.insert(value);
else if (name == "CA") {
if (ca) corrupt();
if (ca) throw corrupt();
// FIXME: allow blank ca or require skipping field?
ca = parseContentAddressOpt(value);
}
Expand All @@ -77,7 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &

if (compression == "") compression = "bzip2";

if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt();
if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt();
}

std::string NarInfo::to_string(const Store & store) const
Expand All @@ -87,11 +86,11 @@ std::string NarInfo::to_string(const Store & store) const
res += "URL: " + url + "\n";
assert(compression != "");
res += "Compression: " + compression + "\n";
assert(fileHash.type == htSHA256);
res += "FileHash: " + fileHash.to_string(Base32, true) + "\n";
assert(fileHash && fileHash->type == htSHA256);
res += "FileHash: " + fileHash->to_string(Base32, true) + "\n";
res += "FileSize: " + std::to_string(fileSize) + "\n";
assert(narHash.type == htSHA256);
res += "NarHash: " + narHash.to_string(Base32, true) + "\n";
assert(narHash && narHash->type == htSHA256);
res += "NarHash: " + narHash->to_string(Base32, true) + "\n";
res += "NarSize: " + std::to_string(narSize) + "\n";

res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
Expand Down
Loading