nixos/testing-python.nix: Add evalTest

This is a decomposition of the testing-python.nix and build-vms.nix
files into modules.

By refactoring the glue, we accomplish the following:

 - NixOS tests can now use `imports` and other module system features.
    - Network-wide test setup can now be reusable; example:
       - A setup with all VMs configured to use a DNS server
       - Split long, slow tests into multiple tests that import a
         common module that has most of the setup.
    - Type checking for the test arguments
    - (TBD) "generated" options reference docs
 - Aspects that had to be wired through all the glue are now in their
   own files.
    - Chief example: interactive.nix.
    - Also: network.nix

In rewriting this, I've generally stuck as close as possible to the
existing code; copying pieces of logic and rewiring them, without
changing the logic itself.

I've made two exceptions to this rule

 - Introduction of `extraDriverArgs` instead of hardcoded
   interactivity logic.

 - Incorporation of NixOS#144110
   in testScript.nix.

I might revert the latter and split it into a new commit.

nixos/lib/testing-python.nix

@@ -166,6 +166,26 @@ rec {
           ${lib.optionalString (interactive) "--add-flags --interactive"}
       '');
 
+  evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; };
+  runTest = module: (evalTest module).config.run;
+
+  testModules = [
+    ./testing/driver.nix
+    ./testing/interactive.nix
+    ./testing/legacy.nix
+    ./testing/meta.nix
+    ./testing/name.nix
+    ./testing/network.nix
+    ./testing/nodes.nix
+    ./testing/run.nix
+    ./testing/testScript.nix
+    {
+      config = {
+        hostPkgs = pkgs;
+      };
+    }
+  ];
+
   # Make a full-blown test
   makeTest =
     { machine ? null

nixos/lib/testing/driver.nix

@@ -0,0 +1,177 @@
+{ config, lib, hostPkgs, ... }:
+let
+  inherit (lib) mkOption types;
+
+  # Reifies and correctly wraps the python test driver for
+  # the respective qemu version and with or without ocr support
+  testDriver = hostPkgs.callPackage ../test-driver {
+    inherit (config) enableOCR extraPythonPackages;
+    qemu_pkg = config.qemu.package;
+    imagemagick_light = hostPkgs.imagemagick_light.override { inherit (hostPkgs) libtiff; };
+    tesseract4 = hostPkgs.tesseract4.override { enableLanguages = [ "eng" ]; };
+  };
+
+
+  vlans = map (m: m.virtualisation.vlans) (lib.attrValues config.nodes);
+  vms = map (m: m.system.build.vm) (lib.attrValues config.nodes);
+
+  nodeHostNames =
+    let
+      nodesList = map (c: c.system.name) (lib.attrValues config.nodes);
+    in
+    nodesList ++ lib.optional (lib.length nodesList == 1 && !lib.elem "machine" nodesList) "machine";
+
+  # TODO: This is an implementation error and needs fixing
+  # the testing famework cannot legitimately restrict hostnames further
+  # beyond RFC1035
+  invalidNodeNames = lib.filter
+    (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
+    nodeHostNames;
+
+  uniqueVlans = lib.unique (builtins.concatLists vlans);
+  vlanNames = map (i: "vlan${toString i}: VLan;") uniqueVlans;
+  machineNames = map (name: "${name}: Machine;") nodeHostNames;
+
+  withChecks =
+    if lib.length invalidNodeNames > 0 then
+      throw ''
+        Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})!
+        All machines are referenced as python variables in the testing framework which will break the
+        script when special characters are used.
+
+        This is an IMPLEMENTATION ERROR and needs to be fixed. Meanwhile,
+        please stick to alphanumeric chars and underscores as separation.
+      ''
+    else
+      lib.warnIf config.skipLint "Linting is disabled";
+
+  driver =
+    hostPkgs.runCommand "nixos-test-driver-${config.name}"
+      {
+        # inherit testName; TODO (roberth): need this?
+        nativeBuildInputs = [
+          hostPkgs.makeWrapper
+        ] ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.mypy ];
+        testScript = config.testScriptString;
+        preferLocalBuild = true;
+        passthru = config.passthru;
+        meta = config.meta // {
+          mainProgram = "nixos-test-driver";
+        };
+      }
+      ''
+        mkdir -p $out/bin
+
+        vmStartScripts=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
+
+        ${lib.optionalString (!config.skipTypeCheck) ''
+          # prepend type hints so the test script can be type checked with mypy
+          cat "${../test-script-prepend.py}" >> testScriptWithTypes
+          echo "${builtins.toString machineNames}" >> testScriptWithTypes
+          echo "${builtins.toString vlanNames}" >> testScriptWithTypes
+          echo -n "$testScript" >> testScriptWithTypes
+
+          cat -n testScriptWithTypes
+
+          # set pythonpath so mypy knows where to find the imports. this requires the py.typed file.
+          export PYTHONPATH='${../test-driver}'
+          mypy  --no-implicit-optional \
+                --pretty \
+                --no-color-output \
+                testScriptWithTypes
+          unset PYTHONPATH
+        ''}
+
+        echo -n "$testScript" >> $out/test-script
+
+        ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
+
+        ${testDriver}/bin/generate-driver-symbols
+        ${lib.optionalString (!config.skipLint) ''
+          PYFLAKES_BUILTINS="$(
+            echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
+            < ${lib.escapeShellArg "driver-symbols"}
+          )" ${hostPkgs.python3Packages.pyflakes}/bin/pyflakes $out/test-script
+        ''}
+
+        # set defaults through environment
+        # see: ./test-driver/test-driver.py argparse implementation
+        wrapProgram $out/bin/nixos-test-driver \
+          --set startScripts "''${vmStartScripts[*]}" \
+          --set testScript "$out/test-script" \
+          --set vlans '${toString vlans}' \
+          ${lib.escapeShellArgs (lib.concatMap (arg: ["--add-flags" arg]) config.extraDriverArgs)}
+      '';
+
+in
+{
+  options = {
+
+    driver = mkOption {
+      description = "Script that runs the test.";
+      type = types.package;
+      defaultText = lib.literalDocBook "set by the test framework";
+    };
+
+    hostPkgs = mkOption {
+      description = "Nixpkgs attrset used outside the nodes.";
+      type = types.raw;
+      example = lib.literalExpression ''
+        import nixpkgs { inherit system config overlays; }
+      '';
+    };
+
+    qemu.package = mkOption {
+      description = "Which qemu package to use.";
+      type = types.package;
+      default = hostPkgs.qemu_test;
+      defaultText = "hostPkgs.qemu_test";
+    };
+
+    enableOCR = mkOption {
+      description = ''
+        Whether to enable Optical Character Recognition functionality for
+        testing graphical programs.
+      '';
+      type = types.bool;
+      default = false;
+    };
+
+    extraPythonPackages = mkOption {
+      description = ''
+        Python packages to add to the test driver.
+
+        The argument is a Python package set, similar to `pkgs.pythonPackages`.
+      '';
+      type = types.functionTo (types.listOf types.package);
+      default = ps: [ ];
+    };
+
+    extraDriverArgs = mkOption {
+      description = ''
+        Extra arguments to pass to the test driver.
+      '';
+      type = types.listOf types.str;
+      default = [];
+    };
+
+    skipLint = mkOption {
+      type = types.bool;
+      default = false;
+    };
+
+    skipTypeCheck = mkOption {
+      type = types.bool;
+      default = false;
+    };
+  };
+
+  config = {
+    _module.args.hostPkgs = config.hostPkgs;
+
+    driver = withChecks driver;
+
+    # make available on the test runner
+    passthru.driver = config.driver;
+  };
+}

nixos/lib/testing/interactive.nix

@@ -0,0 +1,18 @@
+{ config, lib, moduleType, hostPkgs, ... }:
+let
+  inherit (lib) mkOption types;
+in
+{
+  options = {
+    interactive = mkOption {
+      description = "All the same options, but configured for interactive use.";
+      type = moduleType;
+    };
+  };
+
+  config = {
+    interactive.qemu.package = hostPkgs.qemu;
+    interactive.extraDriverArgs = [ "--interactive" ];
+    passthru.driverInteractive = config.interactive.driver;
+  };
+}

nixos/lib/testing/legacy.nix

@@ -0,0 +1,25 @@
+{ config, options, lib, ... }:
+let
+  inherit (lib) mkIf mkOption types;
+in
+{
+  # This needs options.warnings, which we don't have (yet?).
+  # imports = [
+  #   (lib.mkRenamedOptionModule [ "machine" ] [ "nodes" "machine" ])
+  # ];
+
+  options = {
+    machine = mkOption {
+      internal = true;
+      type = types.raw;
+    };
+  };
+
+  config = {
+    nodes = mkIf options.machine.isDefined (
+      lib.warn
+        "In test `${config.name}': The `machine' attribute in NixOS tests (pkgs.nixosTest / make-test-python.nix / testing-python.nix / makeTest) is deprecated. Please set the equivalent `nodes.machine'."
+        { inherit (config) machine; }
+    );
+  };
+}

nixos/lib/testing/meta.nix

@@ -0,0 +1,12 @@
+{ lib, ... }:
+let
+  inherit (lib) types mkOption;
+in
+{
+  options = {
+    meta.maintainers = lib.mkOption {
+      type = types.listOf types.raw;
+      default = [];
+    };
+  };
+}

nixos/lib/testing/name.nix

@@ -0,0 +1,7 @@
+{ lib, ... }:
+{
+  options.name = lib.mkOption {
+    description = "The name of the test.";
+    type = lib.types.str;
+  };
+}

nixos/lib/testing/network.nix

@@ -0,0 +1,75 @@
+{ lib, nodes, ... }:
+
+with lib;
+
+
+let
+  machines = attrNames nodes;
+
+  machinesNumbered = zipLists machines (range 1 254);
+
+  nodes_ = forEach machinesNumbered (m: nameValuePair m.fst
+    [
+      ({ config, nodes, pkgs, ... }:
+        let
+          interfacesNumbered = zipLists config.virtualisation.vlans (range 1 255);
+          interfaces = forEach interfacesNumbered ({ fst, snd }:
+            nameValuePair "eth${toString snd}" {
+              ipv4.addresses =
+                [{
+                  address = "192.168.${toString fst}.${toString m.snd}";
+                  prefixLength = 24;
+                }];
+            });
+
+          networkConfig =
+            {
+              networking.hostName = mkDefault m.fst;
+
+              networking.interfaces = listToAttrs interfaces;
+
+              networking.primaryIPAddress =
+                optionalString (interfaces != [ ]) (head (head interfaces).value.ipv4.addresses).address;
+
+              # Put the IP addresses of all VMs in this machine's
+              # /etc/hosts file.  If a machine has multiple
+              # interfaces, use the IP address corresponding to
+              # the first interface (i.e. the first network in its
+              # virtualisation.vlans option).
+              networking.extraHosts = flip concatMapStrings machines
+                (m':
+                  let config = (getAttr m' nodes).config; in
+                  optionalString (config.networking.primaryIPAddress != "")
+                    ("${config.networking.primaryIPAddress} " +
+                      optionalString (config.networking.domain != null)
+                        "${config.networking.hostName}.${config.networking.domain} " +
+                      "${config.networking.hostName}\n"));
+
+              virtualisation.qemu.options =
+                let qemu-common = import ../qemu-common.nix { inherit lib pkgs; };
+                in
+                flip concatMap interfacesNumbered
+                  ({ fst, snd }: qemu-common.qemuNICFlags snd fst m.snd);
+            };
+
+        in
+        {
+          key = "ip-address";
+          config = networkConfig // {
+            # Expose the networkConfig items for tests like nixops
+            # that need to recreate the network config.
+            system.build.networkConfig = networkConfig;
+          };
+        }
+      )
+    ]);
+
+  extraNodeConfigs = lib.listToAttrs nodes_;
+in
+{
+  config = {
+    defaults = { config, name, ... }: {
+      imports = extraNodeConfigs.${name};
+    };
+  };
+}