Test against stable versions

Add 2.3 to the flake, and adapt the test to also work with 2.3. Tests
fail, which is expected.

Also clean up some things in the test suite more generally.

flake.nix

@@ -3,8 +3,10 @@
 
   inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
+  inputs.nix-2_3-src = { url = "github:nixos/nix/2.3-maintenance"; flake = false; };
+  inputs.nix-2_4 = { url = "github:nixos/nix/2.4-maintenance"; };
 
-  outputs = { self, nixpkgs, lowdown-src }:
+  outputs = { self, nixpkgs, lowdown-src, nix-2_3-src, nix-2_4 }:
 
     let
 
@@ -177,12 +179,7 @@
       testNixVersions = pkgs: client: daemon: with commonDeps pkgs; with pkgs.lib; pkgs.stdenv.mkDerivation {
         NIX_DAEMON_PACKAGE = daemon;
         NIX_CLIENT_PACKAGE = client;
-        name =
-          "nix-tests"
-          + optionalString
-            (versionAtLeast daemon.version "2.4pre20211005" &&
-             versionAtLeast client.version "2.4pre20211005")
-            "-${client.version}-against-${daemon.version}";
+        name = "nix-tests-${client.version}-against-${daemon.version}";
         inherit version;
 
         src = self;
@@ -527,26 +524,45 @@
             '';
         */
 
-        installTests = forAllSystems (system:
-          let pkgs = nixpkgsFor.${system}; in
-          pkgs.runCommand "install-tests" {
-            againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
-            againstCurrentUnstable =
-              # FIXME: temporarily disable this on macOS because of #3605.
-              if system == "x86_64-linux"
-              then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
-              else null;
-            # Disabled because the latest stable version doesn't handle
-            # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
-            # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
-          } "touch $out");
-
+        installTests = let
+          inherit (nixpkgs) lib;
+          forAllPkgs = f: forAllSystems (system: f nixpkgsFor.${system});
+          allTests = rec {
+            againstSelf = forAllPkgs (pkgs: testNixVersions pkgs pkgs.nix pkgs.pkgs.nix);
+
+            # FIXME: temporarily disable this on macOS because of
+            # #3605. We used to null platforms other than
+            # x86_64-linux, but that broke nix flake check when we
+            # exposed these as standalone hydra jobs.
+            againstCurrentUnstable.x86_64-linux = let
+              pkgs = nixpkgsFor.x86_64-linux;
+            in testNixVersions pkgs pkgs.nix pkgs.nixUnstable;
+
+            againstStable_2_3 = forAllPkgs (pkgs: testNixVersions pkgs pkgs.nix
+              (let
+                pkg = (import "${nix-2_3-src}/release.nix" {
+                  nix = nix-2_3-src;
+                  inherit nixpkgs;
+                  officialRelease = true;
+                }).build.${pkgs.stdenv.hostPlatform.system};
+              in pkg // {
+                inherit (lib.strings.parseDrvName pkg.name) version;
+              }));
+            againstStable_2_4 = forAllPkgs (pkgs: testNixVersions pkgs pkgs.nix
+              nix-2_4.packages.${pkgs.stdenv.hostPlatform.system}.nix);
+          };
+          all = forAllSystems (system:
+            nixpkgsFor.${system}.runCommand
+              "install-tests"
+              (lib.mapAttrs (_: v: v.${system} or null) allTests)
+              "touch $out");
+        in allTests // { inherit all; };
       };
 
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
-        installTests = self.hydraJobs.installTests.${system};
+        installTests = self.hydraJobs.installTests.all.${system};
       } // (if system == "x86_64-linux" then {
         dockerImage = self.hydraJobs.dockerImage.${system};
       } else {}));

mk/run_test.sh

@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-set -u
+set -eu -o pipefail
 
 red=""
 green=""
@@ -15,8 +15,12 @@ if [ -t 1 ]; then
     normal=""
 fi
 (cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+
+set +e
 log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
 status=$?
+set -e
+
 if [ $status -eq 0 ]; then
   echo "$post_run_msg [${green}PASS$normal]"
 elif [ $status -eq 99 ]; then

tests/build-dry.sh

@@ -20,7 +20,7 @@ nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
 
 # TODO: XXX: FIXME: #1793
 # Disable this part of the test until the problem is resolved:
-if [ -n "$ISSUE_1795_IS_FIXED" ]; then
+if [ -n "${ISSUE_1795_IS_FIXED-}" ]; then
 clearStore
 clearCache
 

tests/build-remote.sh

@@ -1,13 +1,19 @@
 if ! canUseSandbox; then exit 99; fi
 if ! [[ $busybox =~ busybox ]]; then exit 99; fi
 
+# system-features=... store query param is not supported with 2.3; store
+# features is just in the global settings. A bit tricky to feed different
+# nix.conf files to these ssh stores which are processes spawned by the outer
+# Nix, so maybe we should just backport this feature?
+requireDaemonVersionAtleast "2.4pre"
+
 unset NIX_STORE_DIR
 unset NIX_STATE_DIR
 
 function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; }
 
 EXTRA_SYSTEM_FEATURES=()
-if [[ -n "$CONTENT_ADDRESSED" ]]; then
+if [[ -n "${CONTENT_ADDRESSED-}" ]]; then
     EXTRA_SYSTEM_FEATURES=("ca-derivations")
 fi
 

tests/ca/build-with-garbage-path.sh

@@ -4,7 +4,7 @@
 
 source common.sh
 
-requireDaemonNewerThan "2.4pre20210621"
+requireDaemonVersionAtleast "2.4pre20210621"
 
 # Get the output path of `rootCA`, and put some garbage instead
 outPath="$(nix-build ./content-addressed.nix -A rootCA --no-out-link)"

tests/ca/common.sh

@@ -2,4 +2,6 @@ source ../common.sh
 
 sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
 
+requireDaemonVersionAtleast "2.4pre"
+
 restartDaemon

tests/ca/duplicate-realisation-in-closure.sh

@@ -1,6 +1,6 @@
 source ./common.sh
 
-requireDaemonNewerThan "2.4pre20210625"
+requireDaemonVersionAtleast "2.4pre20210625"
 
 sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
 

tests/ca/post-hook.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-requireDaemonNewerThan "2.4pre20210626"
+requireDaemonVersionAtleast "2.4pre20210626"
 
 sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
 

tests/ca/recursive.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-requireDaemonNewerThan "2.4pre20210623"
+requireDaemonVersionAtleast "2.4pre20210623"
 
 sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
 

tests/common.sh.in

@@ -1,6 +1,6 @@
 set -e
 
-if [[ -z "$COMMON_SH_SOURCED" ]]; then
+if [[ -z "${COMMON_SH_SOURCED-}" ]]; then
 
 COMMON_SH_SOURCED=1
 
@@ -23,7 +23,7 @@ if [[ -n $NIX_STORE ]]; then
 fi
 export _NIX_IN_TEST=$TEST_ROOT/shared
 export _NIX_TEST_NO_LSOF=1
-export NIX_REMOTE=$NIX_REMOTE_
+export NIX_REMOTE=${NIX_REMOTE_-}
 unset NIX_PATH
 export TEST_HOME=$TEST_ROOT/test-home
 export HOME=$TEST_HOME
@@ -90,13 +90,22 @@ startDaemon() {
     # Start the daemon, wait for the socket to appear.  !!!
     # ‘nix-daemon’ should have an option to fork into the background.
     rm -f $NIX_DAEMON_SOCKET_PATH
-    PATH=$DAEMON_PATH nix daemon &
+    if isDaemonVersionAtleast 2.4; then
+        local -a daemonProc=(nix daemon)
+    else
+        local -a daemonProc=(nix-daemon)
+    fi
+    PATH=$DAEMON_PATH "${daemonProc[@]}" \
+        1> >(tee "$TEST_ROOT/daemon-out" >&1) \
+        2> >(tee "$TEST_ROOT/daemon-err" >&2) &
+    pidDaemon=$!
     for ((i = 0; i < 30; i++)); do
         if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then break; fi
         sleep 1
     done
-    pidDaemon=$!
     trap "killDaemon" EXIT
+    # Save for if daemon is killed
+    NIX_REMOTE_OLD=$NIX_REMOTE
     export NIX_REMOTE=daemon
 }
 
@@ -108,30 +117,35 @@ killDaemon() {
     done
     kill -9 $pidDaemon || true
     wait $pidDaemon || true
+    # Restore old nix remote
+    NIX_REMOTE=$NIX_REMOTE_OLD
     trap "" EXIT
 }
 
 restartDaemon() {
   [[ -z "${pidDaemon:-}" ]] && return 0
 
   killDaemon
-  unset NIX_REMOTE
   startDaemon
 }
 
 if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
     _canUseSandbox=1
 fi
 
-isDaemonNewer () {
+isDaemonVersionAtleast () {
   [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0
   local requiredVersion="$1"
   local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3)
-  [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]]
+  # N.B. We might have have written nix.conf to have nix-command yet.
+  [[ $(nix eval \
+    --extra-experimental-features 'nix-command' \
+    --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]]
 }
 
-requireDaemonNewerThan () {
-  isDaemonNewer "$1" || exit 99
+requireDaemonVersionAtleast () {
+  echo "Daemon is too old for this test, skipping..."
+  isDaemonVersionAtleast "$1" || exit 99
 }
 
 canUseSandbox() {

tests/db-migration.sh

@@ -9,7 +9,6 @@ fi
 source common.sh
 
 killDaemon
-unset NIX_REMOTE
 
 # Fill the db using the older Nix
 PATH_WITH_NEW_NIX="$PATH"

tests/fetchurl.sh

@@ -29,20 +29,24 @@ outPath=$(nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file:
 
 cmp $outPath fetchurl.sh
 
-# Test that we can substitute from a different store dir.
 clearStore
 
-other_store=file://$TEST_ROOT/other_store?store=/fnord/store
+# In 2.3 we couldn't do this, and had to rely on --hashed-mirrors ad-hoc thing
+# instead.
+if isDaemonVersionAtleast "2.4pre"; then
+  # Test that we can substitute from a different store dir.
+  other_store=file://$TEST_ROOT/other_store?store=/fnord/store
 
-hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh)
+  hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh)
 
-storePath=$(nix --store $other_store store add-file ./fetchurl.sh)
+  storePath=$(nix --store $other_store store add-file ./fetchurl.sh)
 
-outPath=$(nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
+  outPath=$(nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
 
-# Test hashed mirrors with an SRI hash.
-nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \
-          --no-out-link --substituters $other_store
+  # Test hashed mirrors with an SRI hash.
+  nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \
+            --no-out-link --substituters $other_store
+fi
 
 # Test unpacking a NAR.
 rm -rf $TEST_ROOT/archive