From db2dfde3d2550d1a6462487afe3e3c1ad798822f Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Fri, 25 Jul 2014 18:41:07 -0600 Subject: [PATCH] Rebase secure onto master --- src/libexpr/primops.cc | 4 + src/libmain/shared.cc | 11 ++ src/libstore/build.cc | 227 +++++++++++++++++++++++++++++------ src/libstore/derivations.cc | 8 +- src/libstore/derivations.hh | 6 +- src/libstore/globals.cc | 1 + src/libstore/globals.hh | 4 + src/libstore/hashrewrite.cc | 98 +++++++++++++++ src/libstore/hashrewrite.hh | 7 ++ src/libstore/local-store.cc | 163 ++++++++++++++++++------- src/libstore/misc.cc | 231 +++++++++++++++++++++++++++++++++++- src/libstore/misc.hh | 22 ++++ src/libstore/schema.sql | 15 +++ src/libstore/store-api.cc | 29 ++++- src/libstore/store-api.hh | 58 ++++++++- src/libutil/archive.cc | 18 +++ src/libutil/archive.hh | 4 + src/libutil/pathhash.cc | 50 ++++++++ src/libutil/pathhash.hh | 17 +++ src/libutil/util.cc | 15 ++- src/libutil/util.hh | 8 ++ src/nix-env/nix-env.cc | 11 +- src/nix-store/nix-store.cc | 18 ++- 23 files changed, 923 insertions(+), 102 deletions(-) create mode 100644 src/libstore/hashrewrite.cc create mode 100644 src/libstore/hashrewrite.hh create mode 100644 src/libutil/pathhash.cc create mode 100644 src/libutil/pathhash.hh diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 543266b22aa..eef8cf2272d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -602,6 +602,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * foreach (DerivationOutputs::iterator, i, drv.outputs) if (i->second.path == "") { Path outPath = makeOutputPath(i->first, h, drvName); + /* XXX */ + Path outPath; + PathHash outPathHash; + makeOutputPath(h, drvName, outPath, outPathHash); drv.env[i->first] = outPath; i->second.path = outPath; } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 14263446fe2..c6ca8320c06 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -144,6 +144,17 @@ static void initAndRun(int argc, char * * argv) gettimeofday(&tv, 0); srandom(tv.tv_usec); + /* Set the trust ID to the value of the NIX_USER_ID environment variable, or use the current user name. */ + currentTrustId = getEnv("NIX_USER_ID"); /* !!! dangerous? */ + if (currentTrustId == "") { + SwitchToOriginalUser sw; + uid_t uid = geteuid(); + struct passwd * pw = getpwuid(uid); + if (!pw) throw Error(format("unknown user ID %1%, go away") % uid); + currentTrustId = pw->pw_name; + } + printMsg(lvlError, format("trust ID is `%1%'") % currentTrustId); + /* Process the NIX_LOG_TYPE environment variable. */ string lt = getEnv("NIX_LOG_TYPE"); if (lt != "") setLogType(lt); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 4376a8322c4..9b6b3077d7d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -647,26 +647,6 @@ HookInstance::~HookInstance() ////////////////////////////////////////////////////////////////////// -typedef map HashRewrites; - - -string rewriteHashes(string s, const HashRewrites & rewrites) -{ - foreach (HashRewrites::const_iterator, i, rewrites) { - assert(i->first.size() == i->second.size()); - size_t j = 0; - while ((j = s.find(i->first, j)) != string::npos) { - debug(format("rewriting @ %1%") % j); - s.replace(j, i->second.size(), i->second); - } - } - return s; -} - - -////////////////////////////////////////////////////////////////////// - - typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; class SubstitutionGoal; @@ -757,7 +737,17 @@ class DerivationGoal : public Goal Environment env; /* Hash rewriting. */ + /* Builders build temporary store paths (e.g., `/nix/store/...random-hash...-aterm'), which are subsequently rewritten to actual content-addressable store paths (i.e., the hash part of the store path equals the hash of the contents). */ + + /* The hash rewrite map that rewrites output equivalences occurring + in the command-line arguments and environment variables to the + actual paths to be used. */ + // HashRewrites rewrites; HashRewrites rewritesToTmp, rewritesFromTmp; + /* The map of output equivalence classes to temporary output + paths. */ + // typedef map OutputMap; + // OutputMap tmpOutputs; typedef map RedirectedOutputs; RedirectedOutputs redirectedOutputs; @@ -834,6 +824,9 @@ class DerivationGoal : public Goal /* Return the set of (in)valid paths. */ PathSet checkPathValidity(bool returnValid, bool checkHash); +- PathSet checkPathValidity(bool returnValid); ++ typedef set OutputEqClasses; ++ OutputEqClasses checkOutputValidity(bool returnValid); /* Abort the goal if `path' failed to build. */ bool pathFailed(const Path & path); @@ -971,11 +964,17 @@ void DerivationGoal::haveDerivation() /* Get the derivation. */ drv = derivationFromPath(worker.store, drvPath); +#if 0 foreach (DerivationOutputs::iterator, i, drv.outputs) worker.store.addTempRoot(i->second.path); /* Check what outputs paths are not already valid. */ PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); +#endif + + /* Check for what output path equivalence classes we do not + already have valid, trusted output paths. */ + OutputEqClasses invalidOutputs = checkOutputValidity(false); /* If they are all valid, then we're done. */ if (invalidOutputs.size() == 0 && buildMode == bmNormal) { @@ -988,12 +987,14 @@ void DerivationGoal::haveDerivation() foreach (PathSet::iterator, i, invalidOutputs) if (pathFailed(*i)) return; +#if 0 /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ if (settings.useSubstitutes && !willBuildLocally(drv)) foreach (PathSet::iterator, i, invalidOutputs) addWaitee(worker.makeSubstitutionGoal(*i, buildMode == bmRepair)); +#endif if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); @@ -1022,7 +1023,10 @@ void DerivationGoal::outputsSubstituted() return; } +#if 0 unsigned int nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); +#endif + unsigned int nrInvalid = checkOutputValidity(false).size(); if (buildMode == bmNormal && nrInvalid == 0) { amDone(ecSuccess); return; @@ -1134,11 +1138,13 @@ void DerivationGoal::inputsRealised() /* Gather information necessary for computing the closure and/or running the build hook. */ +#if 0 /* The outputs are referenceable paths. */ foreach (DerivationOutputs::iterator, i, drv.outputs) { debug(format("building path `%1%'") % i->second.path); allPaths.insert(i->second.path); } +#endif /* Determine the full set of input paths. */ @@ -1150,19 +1156,52 @@ void DerivationGoal::inputsRealised() assert(worker.store.isValidPath(i->first)); Derivation inDrv = derivationFromPath(worker.store, i->first); foreach (StringSet::iterator, j, i->second) - if (inDrv.outputs.find(*j) != inDrv.outputs.end()) - computeFSClosure(worker.store, inDrv.outputs[*j].path, inputPaths); - else - throw Error( - format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'") - % drvPath % *j % i->first); + { + OutputEqClass eqClass = findOutputEqClass(inDrv, *j); + PathSet inputs = findTrustedEqClassMembers(eqClass, currentTrustId); + if (inputs.size() == 0) + /* !!! shouldn't happen, except for garbage + collection? */ + throw Error(format("output `%1%' of derivation `%2%' is missing!") + % *j % i->first); + for (PathSet::iterator k = inputs.begin(); k != inputs.end(); ++k) { + rewrites[hashPartOf(eqClass)] = hashPartOf(*k); + computeFSClosure(*k, inputPaths); + } + } } /* Second, the input sources. */ foreach (PathSet::iterator, i, drv.inputSrcs) computeFSClosure(worker.store, *i, inputPaths); + /* There might be equivalence class collisions now. That is, + different input closures might contain different paths from the + *same* output path equivalence class. We should pick one from + each, and rewrite dependent paths. */ + /* Don't forget to apply the rewritten paths to the hash rewrite map that's applied to the environment variables / command-line arguments. Otherwise the builder will still use the unconsolidated paths. */ + Replacements replacements; + inputPaths = consolidatePaths(inputPaths, false, replacements); + + HashRewrites rewrites2; + for (Replacements::iterator i = replacements.begin(); + i != replacements.end(); ++i) + { + printMsg(lvlError, format("HASH REWRITE %1% %2%") + % hashPartOf(i->first).toString() % hashPartOf(i->second).toString()); + rewrites2[hashPartOf(i->first)] = hashPartOf(i->second); + } + + for (HashRewrites::iterator i = rewrites.begin(); + i != rewrites.end(); ++i) + rewrites[i->first] = PathHash(rewriteHashes(i->second.toString(), rewrites2)); + + /* !!! remove, debug only */ + Replacements dummy; + consolidatePaths(inputPaths, true, dummy); + debug(format("added input paths %1%") % showPaths(inputPaths)); + printMsg(lvlError, format("added input paths %1%") % showPaths(inputPaths)); /* !!! */ allPaths.insert(inputPaths.begin(), inputPaths.end()); @@ -1179,13 +1218,16 @@ void DerivationGoal::inputsRealised() } +#if 0 PathSet outputPaths(const DerivationOutputs & outputs) { PathSet paths; + /* XXX */ foreach (DerivationOutputs::const_iterator, i, outputs) paths.insert(i->second.path); return paths; } +#endif static string get(const StringPairs & map, const string & key) @@ -1221,23 +1263,53 @@ void DerivationGoal::tryToBuild() because we're not allowing multi-threading.) If so, put this goal to sleep until another goal finishes, then try again. */ foreach (DerivationOutputs::iterator, i, drv.outputs) + { + /* We direct each output of the derivation to a temporary location + in the Nix store. Afterwards, we move the outputs to their + final, content-addressed location. */ + Path tmpPath = makeRandomStorePath(namePartOf(i->second.eqClass)); + printMsg(lvlError, format("mapping output id `%1%', class `%2%' to `%3%'") + % i->first % i->second.eqClass % tmpPath); + assert(i->second.eqClass.size() == tmpPath.size()); + + debug(format("building path `%1%'") % tmpPath); + + tmpOutputs[i->second.eqClass] = tmpPath; + + /* This is a referenceable path. Make a note of that for when + we are scanning for references in the output. */ + allPaths.insert(tmpPath); + + /* The environment variables and command-line arguments of the + builder refer to the output path equivalence class. Cause + those references to be rewritten to the temporary + locations. */ + rewrites[hashPartOf(i->second.eqClass)] = hashPartOf(tmpPath); + if (pathIsLockedByMe(i->second.path)) { debug(format("putting derivation `%1%' to sleep because `%2%' is locked by another goal") % drvPath % i->second.path); worker.waitForAnyGoal(shared_from_this()); return; } + } /* Obtain locks on all output paths. The locks are automatically released when we exit this function or Nix crashes. If we can't acquire the lock, then continue; hopefully some other goal can start a build, and if not, the main loop will sleep a few seconds and then retry this goal. */ +#if 0 + /* !!! acquire lock on the derivation or something? or on a + pseudo-path representing the output equivalence class? */ + /* TODO: locking is no longer an issue with random temporary paths, but at the cost of having no blocking if we build the same thing twice in parallel. Maybe the "random" path should actually be a hash of the placeholder and the name of the user who started the build. */ if (!outputLocks.lockPaths(outputPaths(drv.outputs), "", false)) { worker.waitForAWhile(shared_from_this()); return; } +#endif +#if 0 /* Now check again whether the outputs are valid. This is because another process may have started building in parallel. After it has finished and released the locks, we can (and should) @@ -1273,6 +1345,7 @@ void DerivationGoal::tryToBuild() acquired the lock. */ foreach (DerivationOutputs::iterator, i, drv.outputs) if (pathFailed(i->second.path)) return; +#endif /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only @@ -1505,6 +1578,9 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { + return rpDecline; + +#if 0 if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; if (!worker.hook) @@ -1580,6 +1656,7 @@ HookReply DerivationGoal::tryBuildHook() % drvPath % drv.platform % logFile); return rpAccept; +#endif } @@ -1600,9 +1677,9 @@ int childEntry(void * arg) void DerivationGoal::startBuilder() { startNest(nest, lvlInfo, format( - buildMode == bmRepair ? "repairing path(s) %1%" : - buildMode == bmCheck ? "checking path(s) %1%" : - "building path(s) %1%") % showPaths(missingPaths)); + buildMode == bmRepair ? "repairing derivation %1%" : + buildMode == bmCheck ? "checking derivation %1%" : + "building derivation %1%") % drvPath); /* Right platform? */ if (!canBuildLocally(drv.platform)) { @@ -1639,8 +1716,9 @@ void DerivationGoal::startBuilder() env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); /* Add all bindings specified in the derivation. */ + /* The temporary output paths, environment variables, and command-line arguments contain placeholder store paths because the fixed paths are no longer known in advance. Therefore, we rewrite them to the actual paths prior to running the builder. */ foreach (StringPairs::iterator, i, drv.env) - env[i->first] = i->second; + env[i->first] = rewriteHashes(i->second, rewrites); /* Create a temporary directory where the build will take place. */ @@ -1901,7 +1979,10 @@ void DerivationGoal::startBuilder() /* Run the builder. */ - printMsg(lvlChatty, format("executing builder `%1%'") % drv.builder); + + /* We also have to rewrite the path to the builder. */ + string builder = rewriteHashes(drv.builder, rewrites); + printMsg(lvlChatty, format("executing builder `%1%'") % builder); /* Create the log file. */ Path logFile = openLogFile(); @@ -2123,10 +2204,11 @@ void DerivationGoal::initChild() /* Fill in the environment. */ Strings envStrs; foreach (Environment::const_iterator, i, env) - envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp)); + envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp)); // rewrites const char * * envArr = strings2CharPtrs(envStrs); - Path program = drv.builder.c_str(); + string builder = rewriteHashes(drv.builder, rewrites); + Path program = builder.c_str(); std::vector args; /* careful with c_str()! */ string user; /* must be here for its c_str()! */ @@ -2154,7 +2236,7 @@ void DerivationGoal::initChild() } /* Fill in the arguments. */ - string builderBasename = baseNameOf(drv.builder); + string builderBasename = baseNameOf(builder); args.push_back(builderBasename.c_str()); foreach (Strings::iterator, i, drv.args) args.push_back(rewriteHashes(*i, rewritesToTmp).c_str()); @@ -2166,7 +2248,7 @@ void DerivationGoal::initChild() inSetup = false; execve(program.c_str(), (char * *) &args[0], (char * *) envArr); - throw SysError(format("executing `%1%'") % drv.builder); + throw SysError(format("executing `%1%'") % builder); } catch (std::exception & e) { writeToStderr("build error: " + string(e.what()) + "\n"); @@ -2211,11 +2293,13 @@ void DerivationGoal::registerOutputs() ValidPathInfos infos; +#if 0 /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ +#endif foreach (DerivationOutputs::iterator, i, drv.outputs) { - Path path = i->second.path; + Path path = tmpOutputs[i->second.eqClass]; if (missingPaths.find(path) == missingPaths.end()) continue; Path actualPath = path; @@ -2319,6 +2403,51 @@ void DerivationGoal::registerOutputs() HashResult hash; PathSet references = scanForReferences(actualPath, allPaths, hash); +// XXX + PathSet referenced; + if (!pathExists(path + "/nix-support/no-scan")) { + Paths referenced2 = filterReferences(path, + Paths(allPaths.begin(), allPaths.end())); + referenced = PathSet(referenced2.begin(), referenced2.end()); + + /* For debugging, print out the referenced and + unreferenced paths. */ + PathSet unreferenced; + insert_iterator ins(unreferenced, unreferenced.begin()); + set_difference( + inputPaths.begin(), inputPaths.end(), + referenced.begin(), referenced.end(), ins); + printMsg(lvlError, format("unreferenced inputs: %1%") % showPaths(unreferenced)); + printMsg(lvlError, format("referenced inputs: %1%") % showPaths(referenced)); + } + + /* Rewrite each output to a name matching its content hash. + I.e., enforce the hash invariant: the hash part of a store + path matches the contents at that path. + + This also registers the final output path as valid, and + sets it references. */ + Path finalPath = addToStore(path, + hashPartOf(path), namePartOf(path), + referenced); + printMsg(lvlError, format("produced final path `%1%'") % finalPath); + + /* Register the fact that this output path is a member of some + output path equivalence class (for a certain user, at + least). This is how subsequent derivations will be able to + find it. */ + Transaction txn; + createStoreTransaction(txn); + addOutputEqMember(txn, i->second.eqClass, currentTrustId, finalPath); + txn.commit(); + + /* Get rid of the temporary output. !!! optimise all this by + *moving* the temporary output to the new location and + applying rewrites in situ. */ + deletePath(path); + } +// End XXX + if (buildMode == bmCheck) { ValidPathInfo info = worker.store.queryPathInfo(path); if (hash.first != info.hash) @@ -2478,6 +2607,7 @@ void DerivationGoal::handleEOF(int fd) } +#if 0 PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { PathSet result; @@ -2490,8 +2620,23 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) } return result; } +#endif +DerivationGoal::OutputEqClasses DerivationGoal::checkOutputValidity(bool returnValid) +{ + OutputEqClasses result; + foreach (DerivationOutputs::iterator, i, drv.outputs) { + PathSet paths = findTrustedEqClassMembers(i->second.eqClass, currentTrustId); + if (paths.size() > 0) { + if (returnValid) result.insert(i->second.eqClass); + } else { + if (!returnValid) result.insert(i->second.eqClass); + } + } + return result; +} + bool DerivationGoal::pathFailed(const Path & path) { if (!settings.cacheFailure) return false; @@ -2535,11 +2680,13 @@ class SubstitutionGoal : public Goal /* The store path that should be realised through a substitute. */ Path storePath; +#if 0 /* The remaining substituters. */ Paths subs; /* The current substituter. */ Path sub; +#endif /* Whether any substituter can realise this path */ bool hasSubstitute; @@ -2655,7 +2802,7 @@ void SubstitutionGoal::tryNext() { trace("trying next substituter"); - if (subs.size() == 0) { + if (true /* !!! subs.size() == 0 */) { /* None left. Terminate this goal and let someone else deal with it. */ debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath); @@ -2666,8 +2813,10 @@ void SubstitutionGoal::tryNext() return; } +#if 0 sub = subs.front(); subs.pop_front(); +#endif SubstitutablePathInfos infos; PathSet dummy(singleton(storePath)); @@ -2711,6 +2860,7 @@ void SubstitutionGoal::referencesValid() void SubstitutionGoal::tryToRun() { +#if 0 trace("trying to run"); /* Make sure that we are allowed to start a build. Note that even @@ -2793,11 +2943,13 @@ void SubstitutionGoal::tryToRun() if (settings.printBuildTrace) printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub); +#endif } void SubstitutionGoal::finished() { +#if 0 trace("substitute finished"); /* Since we got an EOF on the logger pipe, the substitute is @@ -2885,6 +3037,7 @@ void SubstitutionGoal::finished() printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath); amDone(ecSuccess); +#endif } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b452aa2caf5..704430144e9 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -72,7 +72,7 @@ static Derivation parseDerivation(const string & s) while (!endOfList(str)) { DerivationOutput out; expect(str, "("); string id = parseString(str); - expect(str, ","); out.path = parsePath(str); + expect(str, ","); out.eqClass = parseString(str); // !!! parsePath expect(str, ","); out.hashAlgo = parseString(str); expect(str, ","); out.hash = parseString(str); expect(str, ")"); @@ -158,7 +158,7 @@ string unparseDerivation(const Derivation & drv) foreach (DerivationOutputs::const_iterator, i, drv.outputs) { if (first) first = false; else s += ','; s += '('; printString(s, i->first); - s += ','; printString(s, i->second.path); + s += ','; printString(s, i->second.eqClass); s += ','; printString(s, i->second.hashAlgo); s += ','; printString(s, i->second.hash); s += ')'; @@ -239,8 +239,8 @@ Hash hashDerivationModulo(StoreAPI & store, Derivation drv) DerivationOutputs::const_iterator i = drv.outputs.begin(); return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" - + i->second.hash + ":" - + i->second.path); + + i->second.hash /* !!! + ":" + + i->second.path);*/ } /* For other derivations, replace the inputs paths with recursive diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 04b64dfc88a..4769725f0ac 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -17,15 +17,15 @@ const string drvExtension = ".drv"; struct DerivationOutput { - Path path; + OutputEqClass eqClass; string hashAlgo; /* hash used for expected hash computation */ string hash; /* expected hash, may be null */ DerivationOutput() { } - DerivationOutput(Path path, string hashAlgo, string hash) + DerivationOutput(OutputEqClass eqClass, string hashAlgo, string hash) { - this->path = path; + this->eqClass = eqClass; this->hashAlgo = hashAlgo; this->hash = hash; } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 82f3e952a0b..764969e5857 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -35,6 +35,7 @@ Settings::Settings() long res = sysconf(_SC_NPROCESSORS_ONLN); if (res > 0) buildCores = res; #endif + currentTrustId = ""; // XXX readOnlyMode = false; thisSystem = SYSTEM; maxSilentTime = 0; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index f1748336fda..a2ff792f347 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -79,6 +79,10 @@ struct Settings { auto-detected. */ unsigned int buildCores; + /* Current trust ID. !!! Of course, this shouldn't be a global + variable. */ + string currentTrustId; + /* Read-only mode. Don't copy stuff to the store, don't change the database. */ bool readOnlyMode; diff --git a/src/libstore/hashrewrite.cc b/src/libstore/hashrewrite.cc new file mode 100644 index 00000000000..67795fe01a5 --- /dev/null +++ b/src/libstore/hashrewrite.cc @@ -0,0 +1,98 @@ +/* +* Any component in the Nix store resides has a store path name that has a hash component equal to the hash of the contents of that component, i.e., hashPartOf(path) = hashOf(contentsAt(path)). E.g., a path /nix/store/nc35k7yr8...-foo would have content hash nc35k7yr8... When building components in the Nix store, we don't know the content hash until after the component has been built. We handle this by building the component at some randomly generated prefix in the Nix store, and then afterwards *rewriting* the random prefix to the hash of the actual contents. If components that reference themselves, such as ELF executables that contain themselves in their RPATH, we compute content hashes "modulo" the original prefix, i.e., we zero out every occurence of the randomly generated prefix, compute the content hash, then rewrite the random prefix to the final location. +* Take the position of self-references into account when computing content hashes. This is to prevent a rewrite of +...HASH...HASH... +and +...HASH...0000... +(where HASH is the randomly generated prefix) from hashing to the same value. This would happen because they would both resolve to ...0000...0000... Exploiting this into a security hole is left as an exercise to the reader ;-) +*/ + +////////////////////////////////////////////////////////////////////// + + +typedef map HashRewrites; + + +string rewriteHashes(string s, const HashRewrites & rewrites) +{ + foreach (HashRewrites::const_iterator, i, rewrites) { + assert(i->first.size() == i->second.size()); + size_t j = 0; + while ((j = s.find(i->first, j)) != string::npos) { + debug(format("rewriting @ %1%") % j); + s.replace(j, i->second.size(), i->second); + } + } + return s; +} + + +////////////////////////////////////////////////////////////////////// + +typedef map HashRewrites; + +string rewriteHashes(string s, const HashRewrites & rewrites, + vector & positions) +{ + for (HashRewrites::const_iterator i = rewrites.begin(); + i != rewrites.end(); ++i) + { + string from = i->first.toString(), to = i->second.toString(); + + assert(from.size() == to.size()); + + unsigned int j = 0; + while ((j = s.find(from, j)) != string::npos) { + debug(format("rewriting @ %1%") % j); + positions.push_back(j); + s.replace(j, to.size(), to); + j += to.size(); + } + } + + return s; +} + + +string rewriteHashes(const string & s, const HashRewrites & rewrites) +{ + vector dummy; + return rewriteHashes(s, rewrites, dummy); +} + + +static Hash hashModulo(string s, const PathHash & modulus) +{ + vector positions; + + if (!modulus.isNull()) { + /* Zero out occurences of `modulus'. */ + HashRewrites rewrites; + rewrites[modulus] = PathHash(); /* = null hash */ + s = rewriteHashes(s, rewrites, positions); + } + + string positionPrefix; + + for (vector::iterator i = positions.begin(); + i != positions.end(); ++i) + positionPrefix += (format("|%1%") % *i).str(); + + positionPrefix += "||"; + + debug(format("positions %1%") % positionPrefix); + + return hashString(htSHA256, positionPrefix + s); +} + + +static PathSet rewriteReferences(const PathSet & references, + const HashRewrites & rewrites) +{ + PathSet result; + for (PathSet::const_iterator i = references.begin(); i != references.end(); ++i) + result.insert(rewriteHashes(*i, rewrites)); + return result; +} + + diff --git a/src/libstore/hashrewrite.hh b/src/libstore/hashrewrite.hh new file mode 100644 index 00000000000..439b90fb684 --- /dev/null +++ b/src/libstore/hashrewrite.hh @@ -0,0 +1,7 @@ +/* Hash rewriting. */ +typedef map HashRewrites; + +string rewriteHashes(string s, const HashRewrites & rewrites, + vector & positions); + +string rewriteHashes(const string & s, const HashRewrites & rewrites); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7d78ab7ecfa..81f10df2869 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -962,12 +962,76 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) } end_retry_sqlite; } +void addOutputEqMember(const Transaction & txn, + const OutputEqClass & eqClass, const TrustId & trustId, + const Path & path) +{ + OutputEqMembers members; + queryOutputEqMembers(txn, eqClass, members); + + for (OutputEqMembers::iterator i = members.begin(); + i != members.end(); ++i) + if (i->trustId == trustId && i->path == path) return; + + OutputEqMember member; + member.trustId = trustId; + member.path = path; + members.push_back(member); + + Strings ss; + + for (OutputEqMembers::iterator i = members.begin(); + i != members.end(); ++i) + { + Strings ss2; + ss2.push_back(i->trustId); + ss2.push_back(i->path); + ss.push_back(packStrings(ss2)); + } + + nixDB.setStrings(txn, dbEquivalences, eqClass, ss); + + OutputEqClasses classes; + queryOutputEqClasses(txn, path, classes); + + classes.insert(eqClass); + + nixDB.setStrings(txn, dbEquivalenceClasses, path, + Strings(classes.begin(), classes.end())); +} + +void queryOutputEqMembers(const Transaction & txn, + const OutputEqClass & eqClass, OutputEqMembers & members) +{ + Strings ss; + nixDB.queryStrings(txn, dbEquivalences, eqClass, ss); + + for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) { + Strings ss2 = unpackStrings(*i); + if (ss2.size() != 2) continue; + Strings::iterator j = ss2.begin(); + OutputEqMember member; + member.trustId = *j++; + member.path = *j++; + members.push_back(member); + } +} + + +void queryOutputEqClasses(const Transaction & txn, + const Path & path, OutputEqClasses & classes) +{ + Strings ss; + nixDB.queryStrings(txn, dbEquivalenceClasses, path, ss); + classes.insert(ss.begin(), ss.end()); +} +#if 0 Path LocalStore::queryDeriver(const Path & path) { return queryPathInfo(path).deriver; } - +#endif PathSet LocalStore::queryValidDerivers(const Path & path) { @@ -1336,14 +1400,34 @@ void LocalStore::invalidatePath(const Path & path) care of deleting the references entries for `path'. */ } +// _addToStore -> addToStore -> addToStoreFromDump Path LocalStore::addToStoreFromDump(const string & dump, const string & name, - bool recursive, HashType hashAlgo, bool repair) -{ - Hash h = hashString(hashAlgo, dump); - - Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); - +// bool recursive, HashType hashAlgo, bool repair) + const PathHash & selfHash, const PathSet & references) // code written as if recursive=true +{ + /* Hash the contents, modulo the previous hash reference (if it + had one). */ + Hash contentHash = hashModulo(dump, selfHash); + // Hash h = hashString(hashAlgo, dump); + + /* Construct the new store path. */ + Path dstPath; + PathHash pathHash; + makeStorePath(contentHash, name, dstPath, pathHash); + // Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); + + /* If the contents had a previous hash reference, rewrite those + references to the new hash. */ + HashRewrites rewrites; + if (!selfHash.isNull()) { + rewrites[selfHash] = pathHash; + vector positions; + dump = rewriteHashes(dump, rewrites, positions); + /* !!! debug code, remove */ + PathHash contentHash2 = hashModulo(dump, pathHash); + assert(contentHash2 == contentHash); + } addTempRoot(dstPath); if (repair || !isValidPath(dstPath)) { @@ -1378,11 +1462,19 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, optimisePath(dstPath); // FIXME: combine with hashPath() + /* Set the references for the new path. Of course, any + hash rewrites have to be applied to the references, + too. */ + PathSet references2 = rewriteReferences(references, rewrites); + + registerValidPath(dstPath, contentHash, references2, ""); +#if 0 ValidPathInfo info; info.path = dstPath; info.hash = hash.first; info.narSize = hash.second; registerValidPath(info); +#endif } outputLock.setDeletion(true); @@ -1393,7 +1485,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, Path LocalStore::addToStore(const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) + const PathHash & selfHash, const string & suffix, const PathSet & references, const HashRewrites & rewrites) + // bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) { Path srcPath(absPath(_srcPath)); debug(format("adding `%1%' to the store") % srcPath); @@ -1402,53 +1495,31 @@ Path LocalStore::addToStore(const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (recursive) + if (recursive) { + SwitchToOriginalUser sw; dumpPath(srcPath, sink, filter); - else + } else sink.s = readFile(srcPath); - return addToStoreFromDump(sink.s, baseNameOf(srcPath), recursive, hashAlgo, repair); + if (rewrites.size() != 0) sink.s = rewriteHashes(sink.s, rewrites); + + return addToStoreFromDump(sink.s, suffix == "" ? baseNameOf(srcPath) : suffix, + //recursive, hashAlgo, repair); + selfHash, rewriteReferences(references, rewrites)); } Path LocalStore::addTextToStore(const string & name, const string & s, const PathSet & references, bool repair) { - Path dstPath = computeStorePathForText(name, s, references); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { + // Path dstPath = computeStorePathForText(name, s, references); - PathLocks outputLock(singleton(dstPath)); - - if (repair || !isValidPath(dstPath)) { - - if (pathExists(dstPath)) deletePath(dstPath); - - writeFile(dstPath, s); - - canonicalisePathMetaData(dstPath, -1); - - HashResult hash = hashPath(htSHA256, dstPath); - - optimisePath(dstPath); - - ValidPathInfo info; - info.path = dstPath; - info.hash = hash.first; - info.narSize = hash.second; - info.references = references; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; + StringSink sink; + makeSingletonArchive(s, sink); + return addToStoreFromDump(sink.s, name, + PathHash(), references); } - struct HashAndWriteSink : Sink { Sink & writeSink; @@ -1727,6 +1798,7 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) bool errors = false; +#if 0 /* Acquire the global GC lock to prevent a garbage collection. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); @@ -1797,6 +1869,7 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) } } } +#endif return errors; } @@ -1930,6 +2003,7 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) /* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ void LocalStore::upgradeStore6() { +#if 0 printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); openDB(true); @@ -1956,6 +2030,7 @@ void LocalStore::upgradeStore6() std::cerr << "\n"; txn.commit(); +#endif } @@ -1998,9 +2073,11 @@ static void makeMutable(const Path & path) /* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ void LocalStore::upgradeStore7() { +#if 0 if (getuid() != 0) return; printMsg(lvlError, "removing immutable bits from the Nix store (this may take a while)..."); makeMutable(settings.nixStore); +#endif } #else diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 6ecf8787cf6..a85271e2496 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -59,13 +59,240 @@ void computeFSClosure(StoreAPI & store, const Path & path, } -Path findOutput(const Derivation & drv, string id) +OutputEqClass findOutputEqClass(const Derivation & drv, const string & id) { foreach (DerivationOutputs::const_iterator, i, drv.outputs) - if (i->first == id) return i->second.path; + if (i->first == id) return i->second.eqClass; throw Error(format("derivation has no output `%1%'") % id); } +/* Before consolidating/building, consider all trusted paths in the equivalence classes of the input derivations. */ +PathSet findTrustedEqClassMembers(const OutputEqClass & eqClass, + const TrustId & trustId) +{ + OutputEqMembers members; + queryOutputEqMembers(noTxn, eqClass, members); + + PathSet result; + for (OutputEqMembers::iterator j = members.begin(); j != members.end(); ++j) + if (j->trustId == trustId || j->trustId == "root") result.insert(j->path); + + return result; +} + + +Path findTrustedEqClassMember(const OutputEqClass & eqClass, + const TrustId & trustId) +{ + PathSet paths = findTrustedEqClassMembers(eqClass, trustId); + if (paths.size() == 0) + throw Error(format("no output path in equivalence class `%1%' is known") % eqClass); + return *(paths.begin()); +} + + +typedef map ClassMap; +typedef map FinalClassMap; + +static void findBestRewrite(const ClassMap::const_iterator & pos, + const ClassMap::const_iterator & end, + const PathSet & selection, const PathSet & unselection, + unsigned int & bestCost, PathSet & bestSelection) +{ + if (pos != end) { + for (PathSet::iterator i = pos->second.begin(); + i != pos->second.end(); ++i) + { + PathSet selection2(selection); + selection2.insert(*i); + + PathSet unselection2(unselection); + for (PathSet::iterator j = pos->second.begin(); + j != pos->second.end(); ++j) + if (i != j) unselection2.insert(*j); + + ClassMap::const_iterator j = pos; ++j; + findBestRewrite(j, end, selection2, unselection2, + bestCost, bestSelection); + } + return; + } + + PathSet badPaths; + for (PathSet::iterator i = selection.begin(); + i != selection.end(); ++i) + { + PathSet closure; + computeFSClosure(*i, closure); + for (PathSet::iterator j = closure.begin(); + j != closure.end(); ++j) + if (unselection.find(*j) != unselection.end()) + badPaths.insert(*i); + } + + // printMsg(lvlError, format("cost %1% %2%") % badPaths.size() % showPaths(badPaths)); + + if (badPaths.size() < bestCost) { + bestCost = badPaths.size(); + bestSelection = selection; + } +} + + +static Path maybeRewrite(const Path & path, const PathSet & selection, + const FinalClassMap & finalClassMap, const PathSet & sources, + Replacements & replacements, + unsigned int & nrRewrites) +{ + startNest(nest, lvlError, format("considering rewriting `%1%'") % path); + + assert(selection.find(path) != selection.end()); + + if (replacements.find(path) != replacements.end()) return replacements[path]; + + PathSet references; + queryReferences(noTxn, path, references); + + HashRewrites rewrites; + PathSet newReferences; + + for (PathSet::iterator i = references.begin(); i != references.end(); ++i) { + /* Handle sources (which are not in any equivalence class) properly. */ + /* Also ignore self-references. */ + if (*i == path || sources.find(*i) != sources.end()) { + newReferences.insert(*i); + continue; + } + + OutputEqClasses classes; + queryOutputEqClasses(noTxn, *i, classes); + assert(classes.size() > 0); + + FinalClassMap::const_iterator j = finalClassMap.find(*(classes.begin())); + assert(j != finalClassMap.end()); + + Path newPath = maybeRewrite(j->second, selection, + finalClassMap, sources, replacements, nrRewrites); + + if (*i != newPath) + rewrites[hashPartOf(*i)] = hashPartOf(newPath); + + newReferences.insert(newPath); + } + + /* Handle the case where all the direct references of a path are in the selection but some indirect reference isn't (in which case the path should still be rewritten). */ + if (rewrites.size() == 0) { + replacements[path] = path; + return path; + } + + printMsg(lvlError, format("rewriting `%1%'") % path); + + Path newPath = addToStore(path, + hashPartOf(path), namePartOf(path), + references, rewrites); + + /* Set the equivalence class for paths produced through rewriting. */ + /* !!! we don't know which eqClass `path' is in! That is to say, + we don't know which one is intended here. */ + OutputEqClasses classes; + queryOutputEqClasses(noTxn, path, classes); + for (PathSet::iterator i = classes.begin(); i != classes.end(); ++i) { + Transaction txn; + createStoreTransaction(txn); + addOutputEqMember(txn, *i, currentTrustId, newPath); + txn.commit(); + } + + nrRewrites++; + + printMsg(lvlError, format("rewrote `%1%' to `%2%'") % path % newPath); + + replacements[path] = newPath; + + return newPath; +} + +/* +If we do +$ NIX_USER_ID=foo nix-env -i libXext $ NIX_USER_ID=root nix-env -i libXt $ NIX_USER_ID=foo nix-env -i libXmu +(where libXmu depends on libXext and libXt, who both depend on libX11), then the following will happen: +* User foo builds libX11 and libXext because they don't exist yet. +* User root builds libX11 and libXt because the latter doesn't exist yet, while the former *does* exist but cannot be trusted. The instance of libX11 built by root will almost certainly differ from the one built by foo, so they are stored in separate locations. +* User foo builds libXmu, which requires libXext and libXt. Foo has trusted copies of both (libXext was built by himself, while libXt was built by root, who is trusted by foo). So libXmu is built with foo's libXext and root's libXt as inputs. +* The resulting libXmu will link against two copies of libX11, namely the one used by foo's libXext and the one used by root's libXt. This is bad semantically (it's observable behaviour, and might well lead to build time or runtime failure (e.g., duplicate definitions of symbols)) and in terms of efficiency (the closure of libXmu contains two copies of libX11, so both must be deployed). +The problem is to apply hash rewriting to "consolidate" the set of input paths to a build. The invariant we wish to maintain is that any closure may contain at most one path from each equivalence class. +So in the case of a collision, we select one path from each class, and *rewrite* all paths in that set to point only to paths in that set. For instance, in the example above, we can rewrite foo's libXext to link against root's libX11. That is, the hash part of foo's libX11 is replaced by the hash part of root's libX11. +*/ +PathSet consolidatePaths(const PathSet & paths, bool checkOnly, + Replacements & replacements) +{ + printMsg(lvlError, format("consolidating")); + + ClassMap classMap; + PathSet sources; + + for (PathSet::const_iterator i = paths.begin(); i != paths.end(); ++i) { + OutputEqClasses classes; + queryOutputEqClasses(noTxn, *i, classes); + + if (classes.size() == 0) + sources.insert(*i); + else + for (OutputEqClasses::iterator j = classes.begin(); j != classes.end(); ++j) + classMap[*j].insert(*i); + } + + printMsg(lvlError, format("found %1% sources %2%") % sources.size() % showPaths(sources)); + + bool conflict = false; + for (ClassMap::iterator i = classMap.begin(); i != classMap.end(); ++i) + if (i->second.size() >= 2) { + printMsg(lvlError, format("conflict in eq class `%1%'") % i->first); + conflict = true; + } + + if (!conflict) return paths; + + assert(!checkOnly); + + // The hard part is to figure out which path to select from each class. Some selections may be cheaper than others (i.e., require fewer rewrites). + // The current implementation is rather dumb: it tries all possible selections, and picks the cheapest. + // !!! This is an exponential time algorithm. + // There certainly are more efficient common-case (heuristic) approaches. But I don't know yet if there is a worst-case polynomial time algorithm. + const unsigned int infinity = 1000000; + unsigned int bestCost = infinity; + PathSet bestSelection; + findBestRewrite(classMap.begin(), classMap.end(), + PathSet(), PathSet(), bestCost, bestSelection); + + assert(bestCost != infinity); + + printMsg(lvlError, format("cheapest selection %1% %2%") + % bestCost % showPaths(bestSelection)); + + FinalClassMap finalClassMap; + for (ClassMap::iterator i = classMap.begin(); i != classMap.end(); ++i) + for (PathSet::const_iterator j = i->second.begin(); j != i->second.end(); ++j) + if (bestSelection.find(*j) != bestSelection.end()) + finalClassMap[i->first] = *j; + + PathSet newPaths; + unsigned int nrRewrites = 0; + replacements.clear(); + for (PathSet::iterator i = bestSelection.begin(); + i != bestSelection.end(); ++i) + newPaths.insert(maybeRewrite(*i, bestSelection, finalClassMap, + sources, replacements, nrRewrites)); + + newPaths.insert(sources.begin(), sources.end()); + + assert(nrRewrites == bestCost); + + assert(newPaths.size() < paths.size()); + + return newPaths; +} void queryMissing(StoreAPI & store, const PathSet & targets, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh index 144cb7f457c..826e1fc3a70 100644 --- a/src/libstore/misc.hh +++ b/src/libstore/misc.hh @@ -25,6 +25,28 @@ void computeFSClosure(StoreAPI & store, const Path & path, given derivation. */ Path findOutput(const Derivation & drv, string id); +/* Return the output equivalence class denoted by `id' in the + derivation `drv'. */ +OutputEqClass findOutputEqClass(const Derivation & drv, + const string & id); + +/* Return anll trusted path (wrt to the given trust ID) in the given + output path equivalence class, or an empty set if no such paths + currently exist. */ +PathSet findTrustedEqClassMembers(const OutputEqClass & eqClass, + const TrustId & trustId); + +/* Like `findTrustedEqClassMembers', but returns an arbitrary trusted + path, or throws an exception if no such path currently exists. */ +Path findTrustedEqClassMember(const OutputEqClass & eqClass, + const TrustId & trustId); + +typedef map Replacements; + +/* Equivalence class consolidation. This solves the problem that when we combine closures built by different users, the resulting set may contain multiple paths from the same output path equivalence class. */ +PathSet consolidatePaths(const PathSet & paths, bool checkOnly, + Replacements & replacements); + /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index c1b4a689afc..e61b49f1c79 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -28,6 +28,7 @@ create trigger if not exists DeleteSelfRefs before delete on ValidPaths delete from Refs where referrer = old.id and reference = old.id; end; +/* create table if not exists DerivationOutputs ( drv integer not null, id text not null, -- symbolic output id, usually "out" @@ -42,3 +43,17 @@ create table if not exists FailedPaths ( path text primary key not null, time integer not null ); +*/ + +/* dbEquivalences :: OutputEqClass -> [(TrustId, Path)] + + Lists the output paths that have been produced for each extension + class; i.e., the extension of an extension class. */ +static TableId dbEquivalences = 0; + +/* dbEquivalenceClasses :: Path -> [OutputEqClass] + + !!! should be [(TrustId, OutputEqClass)] ? + + Lists for each output path the extension classes that it is in. */ +static TableId dbEquivalenceClasses = 0; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0238e5b0b69..f760f6d63a2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -116,6 +116,7 @@ void checkStoreName(const string & name) name to make sure that changes to either of those are reflected in the hash (e.g. you won't get /nix/store/-name1 and /nix/store/-name2 with equal hash parts). + It does not include the store location in the intensional model. = one of: "text:::..." @@ -165,21 +166,38 @@ void checkStoreName(const string & name) */ +#if 0 Path makeStorePath(const string & type, const Hash & hash, const string & name) +#endif +void makeStorePath(const Hash & contentHash, const string & name, + Path & path, PathHash & pathHash) { + checkStoreName(name); + + /* e.g., "sha256:1abc...:foo.tar.gz" */ + string s = "sha256:" + printHash(contentHash) + ":" + name; +#if 0 /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ string s = type + ":sha256:" + printHash(hash) + ":" + settings.nixStore + ":" + name; +#endif - checkStoreName(name); - - return settings.nixStore + "/" - + printHash32(compressHash(hashString(htSHA256, s), 20)) - + "-" + name; + pathHash = PathHash(hashString(htSHA256, s)); // printHash32(compressHash(hashString(htSHA256, s), 20)) + + path = settings.nixStore + "/" + pathHash.toString() + "-" + name; } +// Random hash generation. +Path makeRandomStorePath(const string & suffix) +{ + Hash hash(htSHA256); + for (unsigned int i = 0; i < hash.hashSize; ++i) + hash.hash[i] = rand() % 256; // !!! improve + return nixStore + "/" + PathHash(hash).toString() + "-" + suffix; +} +#if 0 Path makeOutputPath(const string & id, const Hash & hash, const string & name) { @@ -198,6 +216,7 @@ Path makeFixedOutputPath(bool recursive, printHashType(hashAlgo) + ":" + printHash(hash) + ":"), name); } +#endif std::pair computeStorePathForPath(const Path & srcPath, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b635fee2cf1..d60bd5e46b3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -70,7 +70,38 @@ struct GCResults } }; +/* A trust identifier, which is a name of an entity involved in a + trust relation. Right now this is just a user ID (e.g., + `root'). */ +typedef string TrustId; + +/* An output path equivalence class. They represent outputs of + derivations. That is, a derivation can have several outputs (e.g., + `out', `lib', `man', etc.), each of which maps to a output path + equivalence class. They can map to a number of concrete paths, + depending on what users built the derivation. + + Equivalence classes are actually "placeholder" store paths that + never get built. They do occur in derivations however in + command-line arguments and environment variables, but get + substituted with concrete paths when we actually build. */ +typedef Path OutputEqClass; + +typedef set OutputEqClasses; + + +/* A member of an output path equivalence class, i.e., a store path + that has been produced by a certain derivation. */ +struct OutputEqMember +{ + TrustId trustId; + Path path; +}; + +typedef list OutputEqMembers; + +#if 0 struct SubstitutablePathInfo { Path deriver; @@ -80,6 +111,7 @@ struct SubstitutablePathInfo }; typedef std::map SubstitutablePathInfos; +#endif struct ValidPathInfo @@ -131,9 +163,20 @@ public: virtual void queryReferrers(const Path & path, PathSet & referrers) = 0; + virtual void addOutputEqMember(const Transaction & txn, + const OutputEqClass & eqClass, const TrustId & trustId, + const Path & path); + + virtual void queryOutputEqMembers(const Transaction & txn, + const OutputEqClass & eqClass, OutputEqMembers & members); + + virtual void queryOutputEqClasses(const Transaction & txn, + const Path & path, OutputEqClasses & classes); +#if 0 /* Query the deriver of a store path. Return the empty string if no deriver has been set. */ virtual Path queryDeriver(const Path & path) = 0; +#endif /* Return all currently valid derivations that have `path' as an output. (Note that the result of `queryDeriver()' is the @@ -161,12 +204,17 @@ public: SubstitutablePathInfos & infos) = 0; /* Copy the contents of a path to the store and register the - validity the resulting path. The resulting path is returned. + validity of the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see libutil/archive.hh). */ + virtual Path addToStore(const Path & srcPath, const PathHash & selfHash = PathHash(), + const string & suffix = "", const PathSet & references = PathSet(), + const HashRewrites & rewrites = HashRewrites()); +#if 0 virtual Path addToStore(const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, bool repair = false) = 0; +#endif /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ @@ -282,6 +330,13 @@ Path followLinksToStorePath(const Path & path); /* Constructs a unique store path name. */ +void makeStorePath(const Hash & contentHash, const string & suffix, + Path & path, PathHash & pathHash); + +/* Constructs a random store path name. Only to be used for temporary + build outputs, since these will violate the hash invariant. */ +Path makeRandomStorePath(const string & suffix); +#if 0 Path makeStorePath(const string & type, const Hash & hash, const string & name); @@ -290,6 +345,7 @@ Path makeOutputPath(const string & id, Path makeFixedOutputPath(bool recursive, HashType hashAlgo, Hash hash, string name); +#endif /* This is the preparatory part of addToStore() and addToStoreFixed(); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 5450fd2f718..44a01216dd7 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -131,6 +131,24 @@ void dumpPath(const Path & path, Sink & sink, PathFilter & filter) dump(path, sink, filter); } +void makeSingletonArchive(const string & contents, DumpSink & sink) +{ + /* !!! hacky; have to keep this synchronised with dumpPath(). It + would be better to parameterise dumpPath() with a file system + "traverser". */ + writeString(archiveVersion1, sink); + writeString("(", sink); + writeString("type", sink); + writeString("regular", sink); + + unsigned int size = contents.size(); + writeString("contents", sink); + writeInt(size, sink); + sink((const unsigned char *) contents.c_str(), size); + writePadding(size, sink); + + writeString(")", sink); +} static SerialisationError badArchive(string s) { diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index c216e9768fd..9f086aee573 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -55,6 +55,10 @@ extern PathFilter defaultPathFilter; void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +/* Make an archive consisting of a single non-executable regular + file, with specified string contents. */ +void makeSingletonArchive(const string & contents, DumpSink & sink); + struct ParseSink { virtual void createDirectory(const Path & path) { }; diff --git a/src/libutil/pathhash.cc b/src/libutil/pathhash.cc new file mode 100644 index 00000000000..8371e96ac3d --- /dev/null +++ b/src/libutil/pathhash.cc @@ -0,0 +1,50 @@ +/* Path hashes. */ + +const unsigned int pathHashLen = 32; /* characters */ +const string nullPathHashRef(pathHashLen, 0); + + +PathHash::PathHash() +{ + rep = nullPathHashRef; +} + + +PathHash::PathHash(const Hash & h) +{ + assert(h.type == htSHA256); + rep = printHash32(compressHash(h, 20)); +} + + +PathHash::PathHash(const string & h) +{ + /* !!! hacky; check whether this is a valid 160 bit hash */ + assert(h.size() == pathHashLen); + parseHash32(htSHA1, h); + rep = h; +} + + +string PathHash::toString() const +{ + return rep; +} + + +bool PathHash::isNull() const +{ + return rep == nullPathHashRef; +} + + +bool PathHash::operator ==(const PathHash & hash2) const +{ + return rep == hash2.rep; +} + + +bool PathHash::operator <(const PathHash & hash2) const +{ + return rep < hash2.rep; +} diff --git a/src/libutil/pathhash.hh b/src/libutil/pathhash.hh new file mode 100644 index 00000000000..ec8bffe747b --- /dev/null +++ b/src/libutil/pathhash.hh @@ -0,0 +1,17 @@ +/* Path hashes are the hash components of store paths, e.g., the + `zvhgns772jpj68l40mq1jb74wpfsf0ma' in + `/nix/store/zvhgns772jpj68l40mq1jb74wpfsf0ma-glibc'. These are + truncated SHA-256 hashes of the path contents, */ +struct PathHash +{ +private: + string rep; +public: + PathHash(); + PathHash(const Hash & h); + PathHash(const string & h); + string toString() const; + bool PathHash::isNull() const; + bool operator ==(const PathHash & hash2) const; + bool operator <(const PathHash & hash2) const; +}; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index aa9d99ec335..305ea8cc159 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -152,13 +152,26 @@ string baseNameOf(const Path & path) return string(path, pos + 1); } +PathHash hashPartOf(const Path & path) +{ + assertStorePath(path); + return PathHash(string(path, nixStore.size() + 1, pathHashLen)); +} + + +string namePartOf(const Path & path) +{ + assertStorePath(path); + return string(path, nixStore.size() + 1 + pathHashLen + 1); +} bool isInDir(const Path & path, const Path & dir) { return path[0] == '/' && string(path, 0, dir.size()) == dir && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; + && path[dir.size()] == '/' + && path[dir.size() + 1 + pathHashLen] == '-'; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ad0d377a4f5..47beb65e200 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -46,6 +46,10 @@ Path dirOf(const Path & path); following the final `/'. */ string baseNameOf(const Path & path); +PathHash hashPartOf(const Path & path); + +string namePartOf(const Path & path); + /* Check whether a given path is a descendant of the given directory. */ bool isInDir(const Path & path, const Path & dir); @@ -73,11 +77,13 @@ string readFile(const Path & path, bool drain = false); /* Write a string to a file. */ void writeFile(const Path & path, const string & s); +#if 0 /* Read a line from a file descriptor. */ string readLine(int fd); /* Write a line to a file descriptor. */ void writeLine(int fd, string s); +#endif /* Delete a path; i.e., in the case of a directory, it is deleted recursively. Don't use this at home, kids. The second variant @@ -162,8 +168,10 @@ void writeFull(int fd, const unsigned char * buf, size_t count); MakeError(EndOfFile, Error) +#if 0 /* Read a file descriptor until EOF occurs. */ string drainFD(int fd); +#endif diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 05f6aa35499..664e9ecb01d 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -403,7 +403,10 @@ static void queryInstSources(EvalState & state, if (isDerivation(path)) { elem.setDrvPath(path); - elem.setOutPath(findOutput(derivationFromPath(*store, path), "out")); + elem.setOutPath( + /* XXX check this; may not give a result */ + findTrustedEqClassMember( + findOutputEqClass(derivationFromPath(*store, path), "out"), currentTrustId)); if (name.size() >= drvExtension.size() && string(name, name.size() - drvExtension.size()) == drvExtension) name = string(name, 0, name.size() - drvExtension.size()); @@ -977,7 +980,7 @@ static void opQuery(Globals & globals, /* Query which paths have substitutes. */ - PathSet validPaths, substitutablePaths; + PathSet validPaths /*, substitutablePaths*/; if (printStatus || globals.prebuiltOnly) { PathSet paths; foreach (vector::iterator, i, elems) @@ -988,7 +991,7 @@ static void opQuery(Globals & globals, i->setFailed(); } validPaths = store->queryValidPaths(paths); - substitutablePaths = store->querySubstitutablePaths(paths); +// substitutablePaths = store->querySubstitutablePaths(paths); } @@ -1022,7 +1025,7 @@ static void opQuery(Globals & globals, if (printStatus) { Path outPath = i->queryOutPath(); - bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); + bool hasSubs = false // substitutablePaths.find(outPath) != substitutablePaths.end(); bool isInstalled = installed.find(outPath) != installed.end(); bool isValid = validPaths.find(outPath) != validPaths.end(); if (xmlOutput) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 932789f2c07..32f12772447 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -79,7 +79,8 @@ static PathSet realisePath(Path path, bool build = true) DerivationOutputs::iterator i = drv.outputs.find(*j); if (i == drv.outputs.end()) throw Error(format("derivation `%1%' does not have an output named `%2%'") % p.first % *j); - Path outPath = i->second.path; + Path outPath = findTrustedEqClassMember(findOutputEqClass(drv, drv.name), currentTrustId); + // Path outPath = i->second.path; if (gcRoot == "") printGCWarning(); else { @@ -169,6 +170,7 @@ static void opAdd(Strings opFlags, Strings opArgs) } +#if 0 /* Preload the output of a fixed-output derivation into the Nix store. */ static void opAddFixed(Strings opFlags, Strings opArgs) @@ -213,6 +215,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) makeFixedOutputPath(recursive, hashAlgo, parseHash16or32(hashAlgo, hash), name); } +#endif static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forceRealise) @@ -220,10 +223,14 @@ static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forc if (forceRealise) realisePath(storePath); if (useOutput && isDerivation(storePath)) { Derivation drv = derivationFromPath(*store, storePath); + return findTrustedEqClassMember( + findOutputEqClass(drv, "out"), currentTrustId); +#if 0 PathSet outputs; foreach (DerivationOutputs::iterator, i, drv.outputs) outputs.insert(i->second.path); return outputs; +#endif } else return singleton(storePath); } @@ -313,7 +320,9 @@ static void opQuery(Strings opFlags, Strings opArgs) if (forceRealise) realisePath(*i); Derivation drv = derivationFromPath(*store, *i); foreach (DerivationOutputs::iterator, j, drv.outputs) - cout << format("%1%\n") % j->second.path; + cout << format("%1%\n") % findTrustedEqClassMember( + findOutputEqClass(drv, "out"), + currentTrustId); // j->second.path; } break; } @@ -340,11 +349,14 @@ static void opQuery(Strings opFlags, Strings opArgs) } case qDeriver: +#if 0 foreach (Strings::iterator, i, opArgs) { Path deriver = store->queryDeriver(followLinksToStorePath(*i)); cout << format("%1%\n") % (deriver == "" ? "unknown-deriver" : deriver); } +#endif + assert(0); break; case qBinding: @@ -1031,10 +1043,12 @@ void run(Strings args) op = opRealise; else if (arg == "--add" || arg == "-A") op = opAdd; +#if 0 else if (arg == "--add-fixed") op = opAddFixed; else if (arg == "--print-fixed-path") op = opPrintFixedPath; +#endif else if (arg == "--delete") op = opDelete; else if (arg == "--query" || arg == "-q")