Merge remote-tracking branch 'origin/master' into lazy-trees

.github/workflows/backport.yml

@@ -15,7 +15,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v0.0.7
+        uses: zeebe-io/backport-action@v0.0.8
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

configure.ac

@@ -297,6 +297,17 @@ esac
 AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]),
   sandbox_shell=$withval)
 AC_SUBST(sandbox_shell)
+if test ${cross_compiling:-no} = no && ! test -z ${sandbox_shell+x}; then
+  AC_MSG_CHECKING([whether sandbox-shell has the standalone feature])
+  # busybox shell sometimes allows executing other busybox applets,
+  # even if they are not in the path, breaking our sandbox
+  if PATH= $sandbox_shell -c "busybox" 2>&1 | grep -qv "not found"; then
+    AC_MSG_RESULT(enabled)
+    AC_MSG_ERROR([Please disable busybox FEATURE_SH_STANDALONE])
+  else
+    AC_MSG_RESULT(disabled)
+  fi
+fi
 
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix

doc/manual/src/command-ref/env-common.md

@@ -4,7 +4,7 @@ Most Nix commands interpret the following environment variables:
 
   - `IN_NIX_SHELL`\
     Indicator that tells if the current environment was set up by
-    `nix-shell`. Since Nix 2.0 the values are `"pure"` and `"impure"`
+    `nix-shell`. It can have the values `pure` or `impure`.
 
   - `NIX_PATH`\
     A colon-separated list of directories used to look up Nix

doc/manual/src/contributing/hacking.md

@@ -71,18 +71,6 @@ To install it in `$(pwd)/outputs` and test it:
 nix (Nix) 3.0
 ```
 
-To run a functional test:
-
-```console
-make tests/test-name-should-auto-complete.sh.test
-```
-
-To run the unit-tests for C++ code:
-
-```
-make check
-```
-
 If you have a flakes-enabled Nix you can replace:
 
 ```console
@@ -94,3 +82,29 @@ by:
 ```console
 $ nix develop
 ```
+
+## Testing
+
+Nix comes with three different flavors of tests: unit, functional and integration.
+
+### Unit-tests
+
+The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
+under `src/{library_name}/tests` using the
+[googletest](https://google.github.io/googletest/) framework.
+
+You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option.
+
+### Functional tests
+
+The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
+The whole testsuite can be run with `make install && make installcheck`.
+Individual tests can be run with `make tests/{testName}.sh.test`.
+
+### Integration tests
+
+The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.
+These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup.
+Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
+
+You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`

doc/manual/src/release-notes/rl-next.md

@@ -24,3 +24,19 @@
 
   Selecting derivation outputs using the attribute selection syntax
   (e.g. `nixpkgs#glibc.dev`) no longer works.
+
+* Running nix with the new `--debugger` flag will cause it to start a repl session if
+  there is an exception thrown during eval, or if `builtins.break` is called.  From
+  there one can inspect symbol values and evaluate nix expressions.  In debug mode
+  the following new repl commands are available:
+  ```
+  :env          Show env stack
+  :bt           Show trace stack
+  :st           Show current trace
+  :st <idx>     Change to another trace in the stack
+  :c            Go until end of program, exception, or builtins.break().
+  :s            Go one step
+  ```
+
+* `builtins.fetchTree` (and flake inputs) can now be used to fetch plain files
+  over the `http(s)` and `file` protocols in addition to directory tarballs.

misc/systemd/nix-daemon.service.in

@@ -9,6 +9,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 [Service]
 ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
 KillMode=process
+LimitNOFILE=4096
 
 [Install]
 WantedBy=multi-user.target

scripts/create-darwin-volume.sh

@@ -442,17 +442,21 @@ add_nix_vol_fstab_line() {
     local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}"
     shift
 
-    # wrap `ex` to work around a problem with vim plugins breaking exit codes;
-    # (see https://github.com/NixOS/nix/issues/5468)
-    # we'd prefer EDITOR="/usr/bin/ex --noplugin" but vifs doesn't word-split
+    # wrap `ex` to work around a problem with vim plugins breaking exit codes
+    # (see github.com/NixOS/nix/issues/5468)
+    #
+    # the first draft used `--noplugin`, but github.com/NixOS/nix/issues/6462
+    # suggests we need the less-semantic `-u NONE`
+    #
+    # we'd prefer EDITOR="/usr/bin/ex -u NONE" but vifs doesn't word-split
     # the EDITOR env.
     #
     # TODO: at some point we should switch to `--clean`, but it wasn't added
     # until https://github.com/vim/vim/releases/tag/v8.0.1554 while the macOS
     # minver 10.12.6 seems to have released with vim 7.4
     cat > "$SCRATCH/ex_cleanroom_wrapper" <<EOF
 #!/bin/sh
-/usr/bin/ex --noplugin "\$@"
+/usr/bin/ex -u NONE "\$@"
 EOF
     chmod 755 "$SCRATCH/ex_cleanroom_wrapper"
 
@@ -646,8 +650,9 @@ EOF
         task "Configuring /etc/synthetic.conf to make a mount-point at $NIX_ROOT" >&2
         # technically /etc/synthetic.d/nix is supported in Big Sur+
         # but handling both takes even more code...
+        # Note: `-u NONE` disables vim plugins/rc; see note on --clean earlier
         _sudo "to add Nix to /etc/synthetic.conf" \
-            /usr/bin/ex --noplugin /etc/synthetic.conf <<EOF
+            /usr/bin/ex -u NONE /etc/synthetic.conf <<EOF
 :a
 ${NIX_ROOT:1}
 .
@@ -815,7 +820,8 @@ setup_volume_daemon() {
     local volume_uuid="$2"
     if ! test_voldaemon; then
         task "Configuring LaunchDaemon to mount '$NIX_VOLUME_LABEL'" >&2
-        _sudo "to install the Nix volume mounter" /usr/bin/ex --noplugin "$NIX_VOLUME_MOUNTD_DEST" <<EOF
+        # Note: `-u NONE` disables vim plugins/rc; see note on --clean earlier
+        _sudo "to install the Nix volume mounter" /usr/bin/ex -u NONE "$NIX_VOLUME_MOUNTD_DEST" <<EOF
 :a
 $(generate_mount_daemon "$cmd_type" "$volume_uuid")
 .

src/libcmd/command.cc

@@ -86,6 +86,11 @@ ref<Store> CopyCommand::getDstStore()
 
 EvalCommand::EvalCommand()
 {
+    addFlag({
+        .longName = "debugger",
+        .description = "start an interactive environment if evaluation fails",
+        .handler = {&startReplOnEvalErrors, true},
+    });
 }
 
 EvalCommand::~EvalCommand()
@@ -103,7 +108,7 @@ ref<Store> EvalCommand::getEvalStore()
 
 ref<EvalState> EvalCommand::getEvalState()
 {
-    if (!evalState)
+    if (!evalState) {
         evalState =
             #if HAVE_BOEHMGC
             std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
@@ -113,6 +118,11 @@ ref<EvalState> EvalCommand::getEvalState()
                 searchPath, getEvalStore(), getStore())
             #endif
             ;
+
+        if (startReplOnEvalErrors) {
+            evalState->debugRepl = &runRepl;        
+        };
+    }
     return ref<EvalState>(evalState);
 }
 

src/libcmd/command.hh

@@ -57,6 +57,8 @@ struct CopyCommand : virtual StoreCommand
 
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
+    bool startReplOnEvalErrors = false;
+
     EvalCommand();
 
     ~EvalCommand();
@@ -270,4 +272,8 @@ void printClosureDiff(
     const StorePath & afterPath,
     std::string_view indent);
 
+
+void runRepl(
+    ref<EvalState> evalState,
+    const ValMap & extraEnv);
 }

src/libcmd/installables.cc

@@ -625,11 +625,15 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto drvPath = attr->forceDerivation();
 
     std::set<std::string> outputsToInstall;
+    std::optional<NixInt> priority;
 
-    if (auto aMeta = attr->maybeGetAttr(state->sMeta))
+    if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
         if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
             for (auto & s : aOutputsToInstall->getListOfStrings())
                 outputsToInstall.insert(s);
+        if (auto aPriority = aMeta->maybeGetAttr("priority"))
+            priority = aPriority->getInt();
+    }
 
     if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
         outputsToInstall.clear();
@@ -647,6 +651,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto drvInfo = DerivationInfo {
         .drvPath = std::move(drvPath),
         .outputsToInstall = std::move(outputsToInstall),
+        .priority = priority,
     };
 
     return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)};

src/libcmd/installables.hh

@@ -142,6 +142,7 @@ struct InstallableValue : Installable
     {
         StorePath drvPath;
         std::set<std::string> outputsToInstall;
+        std::optional<NixInt> priority;
     };
 
     virtual std::vector<DerivationInfo> toDerivations() = 0;

src/libcmd/local.mk

@@ -6,9 +6,9 @@ libcmd_DIR := $(d)
 
 libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
-libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
+libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix
 
-libcmd_LDFLAGS += $(LOWDOWN_LIBS) -pthread
+libcmd_LDFLAGS = $(EDITLINE_LIBS) -llowdown -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers