WIP vision for builders data model (build capabilities)

src/libstore/build-capability.cc

@@ -0,0 +1,20 @@
+#include "build-capability.hh"
+#include <algorithm>
+
+namespace nix {
+
+bool
+BuildCapability::canBuild(const Schedulable & schedulable) const
+{
+    return schedulable.getSystem() == system
+        && std::includes(
+            supportedFeatures.begin(), supportedFeatures.end(),
+            schedulable.getRequiredFeatures().begin(), schedulable.getRequiredFeatures().end()
+            )
+        && std::includes(
+            schedulable.getRequiredFeatures().begin(), schedulable.getRequiredFeatures().end(),
+            mandatoryFeatures.begin(), mandatoryFeatures.end()
+            );
+}
+
+} // namespace nix

src/libstore/build-capability.hh

@@ -0,0 +1,78 @@
+#pragma once
+///@file
+
+#include <optional>
+#include <set>
+#include <string>
+
+namespace nix {
+
+class Schedulable {
+public:
+    virtual std::string_view getSystem() const = 0;
+    virtual const std::set<std::string> & getRequiredFeatures() const = 0;
+    virtual bool getPreferLocalBuild() const = 0;
+};
+
+/**
+ * Parameters that determine which derivations can be built.
+ *
+ * *Where* it can be built is determined by context.
+ */
+struct BuildCapability {
+    /**
+     * For a derivation to be buildable by this capability, `system` must match the derivation `system` by case sensitive string equality.
+     *
+     * In a given context, multiple `system`s may be supported. This is represented by having multiple `BuildCapability`s.
+     */
+    std::string system;
+
+    /**
+     * For a derivation to be buildable by this capability, `supportedFeatures` must be a superset of the derivation's `requiredFeatures`, or be equal.
+     */
+    std::set<std::string> supportedFeatures;
+
+    /**
+     * For a derivation to be buildable by this capability, `mandatoryFeatures` must be a subset of the derivation's `requiredFeatures`, or be equal.
+     */
+    std::set<std::string> mandatoryFeatures;
+
+    bool canBuild(const Schedulable & schedulable) const;
+};
+
+/**
+ * Extends `BuildCapability` to include scheduling information.
+ */
+struct SchedulableCapability {
+    /**
+     * Which derivations can be built.
+     */
+    BuildCapability capability;
+
+    /**
+     * An upper bound on the number of derivations that can be built at once.
+     *
+     * If `std::nullopt`, the concurrency is unlimited, or controlled by the remote side.
+     */
+    std::optional<int> maxJobs;
+
+    /**
+     * Whether the capability is local to the current machine.
+     *
+     * This may include VMs that are running on the same machine.
+     * It is the user's responsibility to configure their VMs so that there is no unnecessary copying between VMs.
+     *
+     * This parameter interacts with the `preferLocalBuild` derivation attribute for builds to indicate that the overhead of copying can be expected to be larger than the actual build.
+     */
+    bool isLocal;
+
+    /**
+     * A proportional measure of build performance, typically configured by the user.
+     * Is divided by load to find the best candidate for a build.
+     *
+     * Must be positive.
+     */
+    float speedFactor;
+};
+
+} // namespace nix

src/libstore/machines.cc

@@ -1,4 +1,5 @@
 #include "machines.hh"
+#include "build-capability.hh"
 #include "globals.hh"
 #include "store-api.hh"
 
@@ -7,12 +8,12 @@
 namespace nix {
 
 Machine::Machine(decltype(storeUri) storeUri,
-    decltype(systemTypes) systemTypes,
+     std::set<std::string> systemTypes,
     decltype(sshKey) sshKey,
-    decltype(maxJobs) maxJobs,
-    decltype(speedFactor) speedFactor,
-    decltype(supportedFeatures) supportedFeatures,
-    decltype(mandatoryFeatures) mandatoryFeatures,
+    unsigned int maxJobs,
+    float speedFactor,
+    std::set<std::string> supportedFeatures,
+    std::set<std::string> mandatoryFeatures,
     decltype(sshPublicHostKey) sshPublicHostKey) :
     storeUri(
         // Backwards compatibility: if the URI is schemeless, is not a path,
@@ -29,37 +30,81 @@ Machine::Machine(decltype(storeUri) storeUri,
         || hasPrefix(storeUri, "?")
         ? storeUri
         : "ssh://" + storeUri),
-    systemTypes(systemTypes),
     sshKey(sshKey),
-    maxJobs(maxJobs),
-    speedFactor(speedFactor == 0.0f ? 1.0f : std::move(speedFactor)),
-    supportedFeatures(supportedFeatures),
-    mandatoryFeatures(mandatoryFeatures),
     sshPublicHostKey(sshPublicHostKey)
 {
     if (speedFactor < 0.0)
         throw UsageError("speed factor must be >= 0");
+
+    for (const auto & system : systemTypes) {
+        SchedulableCapability * cap =
+            &capabilities.emplace_back(SchedulableCapability {
+                .capability = BuildCapability {
+                    .system = system,
+                    .supportedFeatures = supportedFeatures,
+                    .mandatoryFeatures = mandatoryFeatures
+                },
+                .maxJobs = maxJobs,
+                .isLocal = false,
+                .speedFactor = speedFactor
+            });
+        capabilitiesBySystem[system].push_back(cap);
+    }
+}
+
+bool Machine::canBuild(const Schedulable & schedulable) const
+{
+    auto system = schedulable.getSystem();
+
+    if (system == "builtin") {
+        // Buildable on any system.
+        return std::any_of(capabilities.begin(), capabilities.end(),
+            [&](const SchedulableCapability & sc) {
+                return sc.capability.canBuild(schedulable);
+            });
+    }
+
+    auto it = capabilitiesBySystem.find(std::string(system));
+    if (it == capabilitiesBySystem.end())
+        return false;
+
+    auto & capsList = it->second;
+
+    return std::any_of(capsList.cbegin(), capsList.cend(),
+        [&](const SchedulableCapability * sc) {
+            return sc->capability.canBuild(schedulable);
+        });
 }
 
 bool Machine::systemSupported(const std::string & system) const
 {
-    return system == "builtin" || (systemTypes.count(system) > 0);
+    return system == "builtin" || capabilitiesBySystem.contains(system);
 }
 
 bool Machine::allSupported(const std::set<std::string> & features) const
 {
-    return std::all_of(features.begin(), features.end(),
-        [&](const std::string & feature) {
-            return supportedFeatures.count(feature) ||
-                mandatoryFeatures.count(feature);
+    // We need to use any_of because this method doesn't know the `system`.
+    // This is not accurate; hence the deprecation.
+    return std::any_of(capabilities.begin(), capabilities.end(),
+        [&](const SchedulableCapability & sc) {
+            return std::all_of(features.begin(), features.end(),
+                [&](const std::string & feature) {
+                    return sc.capability.supportedFeatures.count(feature) ||
+                        sc.capability.mandatoryFeatures.count(feature);
+                });
         });
 }
 
 bool Machine::mandatoryMet(const std::set<std::string> & features) const
 {
-    return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(),
-        [&](const std::string & feature) {
-            return features.count(feature);
+    // We need to use any_of because this method doesn't know the `system`.
+    // This is not accurate; hence the deprecation.
+    return std::any_of(capabilities.begin(), capabilities.end(),
+        [&](const SchedulableCapability & sc) {
+            return std::all_of(sc.capability.mandatoryFeatures.begin(), sc.capability.mandatoryFeatures.end(),
+                [&](const std::string & feature) {
+                    return features.count(feature);
+                });
         });
 }
 
@@ -79,15 +124,16 @@ ref<Store> Machine::openStore() const
     }
 
     {
-        auto & fs = storeParams["system-features"];
-        auto append = [&](auto feats) {
-            for (auto & f : feats) {
-                if (fs.size() > 0) fs += ' ';
-                fs += f;
-            }
-        };
-        append(supportedFeatures);
-        append(mandatoryFeatures);
+        // FIXME
+        // auto & fs = storeParams["system-features"];
+        // auto append = [&](auto feats) {
+        //     for (auto & f : feats) {
+        //         if (fs.size() > 0) fs += ' ';
+        //         fs += f;
+        //     }
+        // };
+        // append(supportedFeatures);
+        // append(mandatoryFeatures);
     }
 
     return nix::openStore(storeUri, storeParams);

src/libstore/machines.hh

@@ -1,48 +1,67 @@
 #pragma once
 ///@file
 
+#include "build-capability.hh"
 #include "types.hh"
 
 namespace nix {
 
 class Store;
 
+struct SchedulableCapability;
+
 struct Machine {
 
     const std::string storeUri;
-    const std::set<std::string> systemTypes;
+
     const std::string sshKey;
-    const unsigned int maxJobs;
-    const float speedFactor;
-    const std::set<std::string> supportedFeatures;
-    const std::set<std::string> mandatoryFeatures;
     const std::string sshPublicHostKey;
+
+    /**
+     * NOTE: The set of capabilities is currently restricted by the constructor
+     *       and the machines format.
+     */
+    std::vector<SchedulableCapability> capabilities;
+
+    /** Index on `capabilities`. Pointers are references into `capabilities`. */
+    std::map<std::string, std::vector<SchedulableCapability *>> capabilitiesBySystem;
+
     bool enabled = true;
 
     /**
+     * @return Whether this host can build the `schedulable`.
+     */
+    bool canBuild(const Schedulable & schedulable) const;
+
+    /**
+     * @deprecated Use `canBuild` instead. This method is not accurate.
+     *
      * @return Whether `system` is either `"builtin"` or in
      * `systemTypes`.
      */
     bool systemSupported(const std::string & system) const;
 
     /**
+     * @deprecated Use `canBuild` instead. This method is not accurate.
+     *
      * @return Whether `features` is a subset of the union of `supportedFeatures` and
      * `mandatoryFeatures`
      */
     bool allSupported(const std::set<std::string> & features) const;
 
     /**
+     * @deprecated Use `canBuild` instead. This method is not accurate.
      * @return @Whether `mandatoryFeatures` is a subset of `features`
      */
     bool mandatoryMet(const std::set<std::string> & features) const;
 
     Machine(decltype(storeUri) storeUri,
-        decltype(systemTypes) systemTypes,
+        std::set<std::string> systemTypes,
         decltype(sshKey) sshKey,
-        decltype(maxJobs) maxJobs,
-        decltype(speedFactor) speedFactor,
-        decltype(supportedFeatures) supportedFeatures,
-        decltype(mandatoryFeatures) mandatoryFeatures,
+        unsigned int maxJobs,
+        float speedFactor,
+        std::set<std::string> supportedFeatures,
+        std::set<std::string> mandatoryFeatures,
         decltype(sshPublicHostKey) sshPublicHostKey);
 
     ref<Store> openStore() const;