Adapt scheduler to work with dynamic derivations

src/libexpr/primops.cc

@@ -1843,7 +1843,7 @@ static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, V
 {
     SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf");
 
-    std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
+    OutputNameView outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
 
     state.mkSingleDerivedPathString(
         SingleDerivedPath::Built {

src/libstore/build/create-derivation-and-realise-goal.cc

@@ -0,0 +1,157 @@
+#include "create-derivation-and-realise-goal.hh"
+#include "worker.hh"
+
+namespace nix {
+
+CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
+    const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
+    : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs })
+    , drvReq(drvReq)
+    , wantedOutputs(wantedOutputs)
+    , buildMode(buildMode)
+{
+    state = &CreateDerivationAndRealiseGoal::getDerivation;
+    name = fmt(
+        "outer obtaining drv from '%s' and then building outputs %s",
+        drvReq->to_string(worker.store),
+        std::visit(overloaded {
+            [&](const OutputsSpec::All) -> std::string {
+                return "* (all of them)";
+            },
+            [&](const OutputsSpec::Names os) {
+                return concatStringsSep(", ", quoteStrings(os));
+            },
+        }, wantedOutputs.raw));
+    trace("created outer");
+
+    worker.updateProgress();
+}
+
+
+CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal()
+{
+}
+
+
+static StorePath pathPartOfReq(const SingleDerivedPath & req)
+{
+    return std::visit(overloaded {
+        [&](const SingleDerivedPath::Opaque & bo) {
+            return bo.path;
+        },
+        [&](const SingleDerivedPath::Built & bfd) {
+            return pathPartOfReq(*bfd.drvPath);
+        },
+    }, req.raw());
+}
+
+
+std::string CreateDerivationAndRealiseGoal::key()
+{
+    /* Ensure that derivations get built in order of their name,
+       i.e. a derivation named "aardvark" always comes before "baboon". And
+       substitution goals and inner derivation goals always happen before
+       derivation goals (due to "b$"). */
+    return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
+}
+
+
+void CreateDerivationAndRealiseGoal::timedOut(Error && ex)
+{
+}
+
+
+void CreateDerivationAndRealiseGoal::work()
+{
+    (this->*state)();
+}
+
+
+void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs)
+{
+    /* If we already want all outputs, there is nothing to do. */
+    auto newWanted = wantedOutputs.union_(outputs);
+    bool needRestart = !newWanted.isSubsetOf(wantedOutputs);
+    wantedOutputs = newWanted;
+
+    if (!needRestart) return;
+
+    if (!optDrvPath)
+        // haven't started steps where the outputs matter yet
+        return;
+    worker.makeDerivationGoal(*optDrvPath, outputs, buildMode);
+}
+
+
+void CreateDerivationAndRealiseGoal::getDerivation()
+{
+    trace("outer init");
+
+    /* The first thing to do is to make sure that the derivation
+       exists.  If it doesn't, it may be created through a
+       substitute. */
+    if (auto optDrvPath = [this]() -> std::optional<StorePath> {
+        if (buildMode != bmNormal) return std::nullopt;
+
+        auto drvPath = StorePath::dummy;
+        try {
+            drvPath = resolveDerivedPath(worker.store, *drvReq);
+        } catch (MissingRealisation) {
+            return std::nullopt;
+        }
+        return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath)
+            ? std::optional { drvPath }
+            : std::nullopt;
+    }()) {
+        trace(fmt("already have drv '%s' for '%s', can go straight to building",
+            worker.store.printStorePath(*optDrvPath),
+            drvReq->to_string(worker.store)));
+
+        loadAndBuildDerivation();
+    } else {
+        trace("need to obtain drv we want to build");
+
+        addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq)));
+
+        state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation;
+        if (waitees.empty()) work();
+    }
+}
+
+
+void CreateDerivationAndRealiseGoal::loadAndBuildDerivation()
+{
+    trace("outer load and build derivation");
+
+    if (nrFailed != 0) {
+        amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
+        return;
+    }
+
+    StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
+    /* Build this step! */
+    concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode);
+    addWaitee(upcast_goal(concreteDrvGoal));
+    state = &CreateDerivationAndRealiseGoal::buildDone;
+    optDrvPath = std::move(drvPath);
+    if (waitees.empty()) work();
+}
+
+
+void CreateDerivationAndRealiseGoal::buildDone()
+{
+    trace("outer build done");
+
+    buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built {
+        .drvPath = drvReq,
+        .outputs = wantedOutputs,
+    });
+
+    if (buildResult.success())
+        amDone(ecSuccess);
+    else
+        amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store)));
+}
+
+
+}

src/libstore/build/create-derivation-and-realise-goal.hh

@@ -0,0 +1,96 @@
+#pragma once
+
+#include "parsed-derivations.hh"
+#include "lock.hh"
+#include "store-api.hh"
+#include "pathlocks.hh"
+#include "goal.hh"
+
+namespace nix {
+
+struct DerivationGoal;
+
+/**
+ * This goal type is essentially the serial composition (like function
+ * composition) of a goal for getting a derivation, and then a
+ * `DerivationGoal` using the newly-obtained derivation.
+ *
+ * In the (currently experimental) general inductive case of derivations
+ * that are themselves build outputs, that first goal will be *another*
+ * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case
+ * where the derivation has no provence and is just referred to by
+ * (content-addressed) store path, that first goal is a
+ * `SubstitutionGoal`.
+ *
+ * If we already have the derivation (e.g. if the evalutator has created
+ * the derivation locally and then instructured the store to build it),
+ * we can skip the first goal entirely as a small optimization.
+ */
+struct CreateDerivationAndRealiseGoal : public Goal
+{
+    /**
+     * How to obtain a store path of the derivation to build.
+     */
+    ref<SingleDerivedPath> drvReq;
+
+    /**
+     * The path of the derivation, once obtained.
+     **/
+    std::optional<StorePath> optDrvPath;
+
+    /**
+     * The goal for the corresponding concrete derivation.
+     **/
+    std::shared_ptr<DerivationGoal> concreteDrvGoal;
+
+    /**
+     * The specific outputs that we need to build.
+     */
+    OutputsSpec wantedOutputs;
+
+    typedef void (CreateDerivationAndRealiseGoal::*GoalState)();
+    GoalState state;
+
+    /**
+     * The final output paths of the build.
+     *
+     * - For input-addressed derivations, always the precomputed paths
+     *
+     * - For content-addressed derivations, calcuated from whatever the
+     *   hash ends up being. (Note that fixed outputs derivations that
+     *   produce the "wrong" output still install that data under its
+     *   true content-address.)
+     */
+    OutputPathMap finalOutputs;
+
+    BuildMode buildMode;
+
+    CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
+        const OutputsSpec & wantedOutputs, Worker & worker,
+        BuildMode buildMode = bmNormal);
+    virtual ~CreateDerivationAndRealiseGoal();
+
+    void timedOut(Error && ex) override;
+
+    std::string key() override;
+
+    void work() override;
+
+    /**
+     * Add wanted outputs to an already existing derivation goal.
+     */
+    void addWantedOutputs(const OutputsSpec & outputs);
+
+    /**
+     * The states.
+     */
+    void getDerivation();
+    void loadAndBuildDerivation();
+    void buildDone();
+
+    JobCategory jobCategory() const override {
+        return JobCategory::Administration;
+    };
+};
+
+}

src/libstore/build/derivation-goal.cc

@@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
-    state = &DerivationGoal::getDerivation;
+    state = &DerivationGoal::loadDerivation;
     name = fmt(
         "building of '%s' from .drv file",
         DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
@@ -164,24 +164,6 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
 }
 
 
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
 void DerivationGoal::loadDerivation()
 {
     trace("loading derivation");
@@ -380,7 +362,12 @@ void DerivationGoal::gaveUpOnSubstitution()
                         worker.store.printStorePath(i.first));
             }
 
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
+            addWaitee(worker.makeGoal(
+                DerivedPath::Built {
+                    .drvPath = makeConstantStorePathRef(i.first),
+                    .outputs = i.second,
+                },
+                buildMode == bmRepair ? bmRepair : bmNormal));
         }
 
     /* Copy the input sources from the eval store to the build
@@ -452,7 +439,12 @@ void DerivationGoal::repairClosure()
         if (drvPath2 == outputsToDrv.end())
             addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
         else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair));
+            addWaitee(worker.makeGoal(
+                DerivedPath::Built {
+                    .drvPath = makeConstantStorePathRef(drvPath2->second),
+                    .outputs = OutputsSpec::All { },
+                },
+                bmRepair));
     }
 
     if (waitees.empty()) {
@@ -1483,7 +1475,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
     if (!useDerivation) return;
     auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-    auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
+    auto * dg = tryGetConcreteDrvGoal(waitee);
     if (!dg) return;
 
     auto outputs = fullDrv.inputDrvs.find(dg->drvPath);

src/libstore/build/derivation-goal.hh

@@ -50,6 +50,13 @@ struct InitialOutput {
     std::optional<InitialOutputStatus> known;
 };
 
+/**
+ * A goal for building some or all of the outputs of a derivation.
+ *
+ * The derivation must already be present, either in the store in a drv
+ * or in memory. If the derivation itself needs to be gotten first, a
+ * `CreateDerivationAndRealiseGoal` goal must be used instead.
+ */
 struct DerivationGoal : public Goal
 {
     /**
@@ -66,8 +73,7 @@ struct DerivationGoal : public Goal
     std::shared_ptr<DerivationGoal> resolvedDrvGoal;
 
     /**
-     * The specific outputs that we need to build.  Empty means all of
-     * them.
+     * The specific outputs that we need to build.
      */
     OutputsSpec wantedOutputs;
 
@@ -229,7 +235,6 @@ struct DerivationGoal : public Goal
     /**
      * The states.
      */
-    void getDerivation();
     void loadDerivation();
     void haveDerivation();
     void outputsSubstitutionTried();
@@ -334,7 +339,9 @@ struct DerivationGoal : public Goal
 
     StorePathSet exportReferences(const StorePathSet & storePaths);
 
-    JobCategory jobCategory() override { return JobCategory::Build; };
+    JobCategory jobCategory() const override {
+        return JobCategory::Build;
+    };
 };
 
 MakeError(NotDeterministic, BuildError);

src/libstore/build/drv-output-substitution-goal.hh

@@ -73,7 +73,9 @@ public:
     void work() override;
     void handleEOF(int fd) override;
 
-    JobCategory jobCategory() override { return JobCategory::Substitution; };
+    JobCategory jobCategory() const override {
+        return JobCategory::Substitution;
+    };
 };
 
 }