Make the Derived Path family of types inductive for dynamic derivations

We want to be able to write down `foo.drv^bar.drv^baz`:
`foo.drv^bar.drv` is the dynamic derivation (since it is itself a
derivation output, `bar.drv` from `foo.drv`).

To that end, we create `Single{Derivation,BuiltPath}` types, that are
very similar except instead of having multiple outputs (in a set or
map), they have a single one. This is for everything to the left of the
rightmost `^`.

`NixStringContextElem` has an analogous change, and now can reuse
`SingleDerivedPath` at the top level. In fact, if we ever get rid of
`DrvDeep`, `NixStringContextElem` could be replaced with
`SingleDerivedPath` entirely!

Important note: some JSON formats have changed.

We already can *produce* dynamic derivations, but we can't refer to them
directly. Today, we can merely express building or example at the top
imperatively over time by building `foo.drv^bar.drv`, and then with a
second nix invocation doing `<result-from-first>^baz`, but this is not
declarative. The ethos of Nix of being able to write down the full plan
everything you want to do, and then execute than plan with a single
command, and for that we need the new inductive form of these types.

src/build-remote/build-remote.cc

@@ -322,7 +322,12 @@ static int main_build_remote(int argc, char * * argv)
                 throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
         } else {
             copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
-            auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } });
+            auto res = sshStore->buildPathsWithResults({
+                DerivedPath::Built {
+                    .drvPath = staticDrvReq(*drvPath),
+                    .outputs = OutputsSpec::All {},
+                }
+            });
             // One path to build should produce exactly one build result
             assert(res.size() == 1);
             optResult = std::move(res[0]);

src/libcmd/installable-attr-path.cc

@@ -92,7 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
     for (auto & [drvPath, outputs] : byDrvPath)
         res.push_back({
             .path = DerivedPath::Built {
-                .drvPath = drvPath,
+                .drvPath = staticDrvReq(drvPath),
                 .outputs = outputs,
             },
             .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value {

src/libcmd/installable-derived-path.cc

@@ -16,18 +16,24 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths()
     }};
 }
 
-std::optional<StorePath> InstallableDerivedPath::getStorePath()
+template<typename DP>
+static StorePath getStorePathFrom(const DP & derivedPath)
 {
     return std::visit(overloaded {
-        [&](const DerivedPath::Built & bfd) {
-            return bfd.drvPath;
+        [&](const typename DP::Built & bfd) {
+            return getStorePathFrom<SingleDerivedPath>(*bfd.drvPath);
         },
-        [&](const DerivedPath::Opaque & bo) {
+        [&](const typename DP::Opaque & bo) {
             return bo.path;
         },
     }, derivedPath.raw());
 }
 
+std::optional<StorePath> InstallableDerivedPath::getStorePath()
+{
+    return getStorePathFrom<DerivedPath>(derivedPath);
+}
+
 InstallableDerivedPath InstallableDerivedPath::parse(
     ref<Store> store,
     std::string_view prefix,
@@ -42,7 +48,7 @@ InstallableDerivedPath InstallableDerivedPath::parse(
             // Remove this prior to stabilizing the new CLI.
             if (storePath.isDerivation()) {
                 auto oldDerivedPath = DerivedPath::Built {
-                    .drvPath = storePath,
+                    .drvPath = staticDrvReq(storePath),
                     .outputs = OutputsSpec::All { },
                 };
                 warn(
@@ -55,8 +61,10 @@ InstallableDerivedPath InstallableDerivedPath::parse(
         },
         // If the user did use ^, we just do exactly what is written.
         [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
+            auto drv = make_ref<SingleDerivedPath>(SingleDerivedPath::parse(*store, prefix));
+            drvRequireExperiment(*drv);
             return DerivedPath::Built {
-                .drvPath = store->parseStorePath(prefix),
+                .drvPath = std::move(drv),
                 .outputs = outputSpec,
             };
         },

src/libcmd/installable-flake.cc

@@ -118,7 +118,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
 
     return {{
         .path = DerivedPath::Built {
-            .drvPath = std::move(drvPath),
+            .drvPath = staticDrvReq(std::move(drvPath)),
             .outputs = std::visit(overloaded {
                 [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
                     std::set<std::string> outputsToInstall;

src/libcmd/installable-value.cc

@@ -55,7 +55,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
 
     else if (v.type() == nString) {
         return {{
-            .path = state->coerceToDerivedPath(pos, v, errorCtx),
+            .path = DerivedPath::fromSingle(
+                state->coerceToSingleDerivedPath(pos, v, errorCtx)),
             .info = make_ref<ExtraPathInfo>(),
         }};
     }

src/libcmd/installables.cc

@@ -514,6 +514,61 @@ ref<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
+static StorePath getBuiltPathFromRealisation(
+    Store & store,
+    const std::map<std::string, Hash> & outputHashes,
+    const DerivationOutputsAndOptPaths & drvOutputs,
+    const StorePath & drvPath,
+    const std::string & output)
+{
+    if (!outputHashes.count(output))
+        throw Error(
+            "the derivation '%s' doesn't have an output named '%s'",
+            store.printStorePath(drvPath), output);
+    if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+        auto outputId =
+            DrvOutput{outputHashes.at(output), output};
+        auto realisation =
+            store.queryRealisation(outputId);
+        if (!realisation)
+            throw Error(
+                "cannot operate on an output of unbuilt "
+                "content-addressed derivation '%s'",
+                outputId.to_string());
+        return realisation->outPath;
+    } else {
+        // If ca-derivations isn't enabled, assume that
+        // the output path is statically known.
+        assert(drvOutputs.count(output));
+        assert(drvOutputs.at(output).second);
+        return *drvOutputs.at(output).second;
+    }
+}
+
+static SingleBuiltPath getBuiltPath(ref<Store> evalStore, ref<Store> store, const SingleDerivedPath & b)
+{
+    return std::visit(
+        overloaded{
+            [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath {
+                return SingleBuiltPath::Opaque { bo.path };
+            },
+            [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath {
+                auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath);
+                auto drv = evalStore->readDerivation(drvPath.outPath());
+                auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
+                auto drvOutputs = drv.outputsAndOptPaths(*store);
+                return SingleBuiltPath::Built {
+                    .drvPath = make_ref<SingleBuiltPath>(std::move(drvPath)),
+                    .output = {
+                        bfd.output,
+                        getBuiltPathFromRealisation(*store, outputHashes, drvOutputs, drvPath.outPath(), bfd.output),
+                    },
+                };
+            },
+        },
+        b.raw());
+}
+
 std::vector<BuiltPathWithResult> Installable::build(
     ref<Store> evalStore,
     ref<Store> store,
@@ -567,7 +622,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                     [&](const DerivedPath::Built & bfd) {
                         auto outputs = resolveDerivedPath(*store, bfd, &*evalStore);
                         res.push_back({aux.installable, {
-                            .path = BuiltPath::Built { bfd.drvPath, outputs },
+                            .path = BuiltPath::Built {
+                                .drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)),
+                                .outputs = outputs,
+                             },
                             .info = aux.info}});
                     },
                     [&](const DerivedPath::Opaque & bo) {
@@ -596,7 +654,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                         for (auto & [outputName, realisation] : buildResult.builtOutputs)
                             outputs.emplace(outputName, realisation.outPath);
                         res.push_back({aux.installable, {
-                            .path = BuiltPath::Built { bfd.drvPath, outputs },
+                            .path = BuiltPath::Built {
+                                .drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)),
+                                .outputs = outputs,
+                            },
                             .info = aux.info,
                             .result = buildResult}});
                     },
@@ -690,7 +751,14 @@ StorePathSet Installable::toDerivations(
                         : throw Error("argument '%s' did not evaluate to a derivation", i->what()));
                 },
                 [&](const DerivedPath::Built & bfd) {
-                    drvPaths.insert(bfd.drvPath);
+                    drvPaths.insert(std::visit(overloaded {
+                        [&](const SingleDerivedPath::Opaque o) -> StorePath {
+                            return o.path;
+                        },
+                        [&](const SingleDerivedPath::Built b) -> StorePath {
+                            throw Error("argument '%s' did not evaluate to output of dynamic derivation '%s' which is not yet built.", i->what(), b.to_string(*store));
+                        },
+                    }, tryResolveDerivedPath(*store, *bfd.drvPath).raw()));
                 },
             }, b.path.raw());
 

src/libcmd/repl.cc

@@ -647,7 +647,7 @@ bool NixRepl::processLine(std::string line)
         if (command == ":b" || command == ":bl") {
             state->store->buildPaths({
                 DerivedPath::Built {
-                    .drvPath = drvPath,
+                    .drvPath = staticDrvReq(drvPath),
                     .outputs = OutputsSpec::All { },
                 },
             });

src/libexpr/eval-cache.cc

@@ -599,7 +599,10 @@ string_t AttrCursor::getStringWithContext()
                             return d.drvPath;
                         },
                         [&](const NixStringContextElem::Built & b) -> const StorePath & {
-                            return b.drvPath;
+                            if (auto * drvPath = std::get_if<SingleDerivedPath::Opaque>(&*b.drvPath))
+                                return drvPath->path;
+                            else
+                                root->state.error("Dynamic derivations in string contexts are not yet supported by the evaluation cache").debugThrow<UnimplementedError>();
                         },
                         [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
                             return o.path;

src/libexpr/eval.cc

@@ -1062,7 +1062,7 @@ void EvalState::mkOutputString(
             : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(),
         NixStringContext {
             NixStringContextElem::Built {
-                .drvPath = drvPath,
+                .drvPath = staticDrvReq(drvPath),
                 .output = outputName,
             }
         });
@@ -2319,7 +2319,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
 }
 
 
-std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
+std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
 {
     NixStringContext context;
     auto s = forceString(v, context, pos, errorCtx);
@@ -2330,21 +2330,16 @@ std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked
             s, csize)
             .withTrace(pos, errorCtx).debugThrow<EvalError>();
     auto derivedPath = std::visit(overloaded {
-        [&](NixStringContextElem::Opaque && o) -> DerivedPath {
-            return DerivedPath::Opaque {
-                .path = std::move(o.path),
-            };
+        [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath {
+            return std::move(o);
         },
-        [&](NixStringContextElem::DrvDeep &&) -> DerivedPath {
+        [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath {
             error(
                 "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
                 s).withTrace(pos, errorCtx).debugThrow<EvalError>();
         },
-        [&](NixStringContextElem::Built && b) -> DerivedPath {
-            return DerivedPath::Built {
-                .drvPath = std::move(b.drvPath),
-                .outputs = OutputsSpec::Names { std::move(b.output) },
-            };
+        [&](NixStringContextElem::Built && b) -> SingleDerivedPath {
+            return std::move(b);
         },
     }, ((NixStringContextElem &&) *context.begin()).raw());
     return {
@@ -2354,38 +2349,41 @@ std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked
 }
 
 
-DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
+SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
 {
-    auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx);
+    auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx);
     auto s = s_;
     std::visit(overloaded {
-        [&](const DerivedPath::Opaque & o) {
+        [&](const SingleDerivedPath::Opaque & o) {
             auto sExpected = store->printStorePath(o.path);
             if (s != sExpected)
                 error(
                     "path string '%s' has context with the different path '%s'",
                     s, sExpected)
                     .withTrace(pos, errorCtx).debugThrow<EvalError>();
         },
-        [&](const DerivedPath::Built & b) {
-            // TODO need derived path with single output to make this
-            // total. Will add as part of RFC 92 work and then this is
-            // cleaned up.
-            auto output = *std::get<OutputsSpec::Names>(b.outputs).begin();
-
-            auto drv = store->readDerivation(b.drvPath);
-            auto i = drv.outputs.find(output);
-            if (i == drv.outputs.end())
-                throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output);
-            auto optOutputPath = i->second.path(*store, drv.name, output);
-            // This is testing for the case of CA derivations
-            auto sExpected = optOutputPath
-                ? store->printStorePath(*optOutputPath)
-                : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render();
+        [&](const SingleDerivedPath::Built & b) {
+            auto placeholder = DownstreamPlaceholder::fromSingleDerivedPathBuilt(b);
+            auto sExpected = std::visit(overloaded {
+                [&](const SingleDerivedPath::Opaque & o) {
+                    auto drv = store->readDerivation(o.path);
+                    auto i = drv.outputs.find(b.output);
+                    if (i == drv.outputs.end())
+                        throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
+                    auto optOutputPath = i->second.path(*store, drv.name, b.output);
+                    // This is testing for the case of CA derivations
+                    return optOutputPath
+                        ? store->printStorePath(*optOutputPath)
+                        : placeholder.render();
+                },
+                [&](const SingleDerivedPath::Built & o) {
+                    return placeholder.render();
+                },
+            }, b.drvPath->raw());
             if (s != sExpected)
                 error(
                     "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
-                    s, output, store->printStorePath(b.drvPath), sExpected)
+                    s, b.output, b.drvPath->to_string(*store), sExpected)
                     .withTrace(pos, errorCtx).debugThrow<EvalError>();
         }
     }, derivedPath.raw());

src/libexpr/eval.hh

@@ -21,7 +21,7 @@ namespace nix {
 class Store;
 class EvalState;
 class StorePath;
-struct DerivedPath;
+struct SingleDerivedPath;
 enum RepairFlag : bool;
 
 
@@ -475,12 +475,12 @@ public:
     StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);
 
     /**
-     * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only.
+     * Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only.
      */
-    std::pair<DerivedPath, std::string_view> coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);
+    std::pair<SingleDerivedPath, std::string_view> coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);
 
     /**
-     * Coerce to `DerivedPath`.
+     * Coerce to `SingleDerivedPath`.
      *
      * Must be a string which is either a literal store path or a
      * "placeholder (see `DownstreamPlaceholder`).
@@ -494,7 +494,7 @@ public:
      * source of truth, and ultimately tells us what we want, and then
      * we ensure the string corresponds to it.
      */
-    DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
+    SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
 
 public: