From 1f88199285cdd99a8c38fb2ec940ece845c70b86 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 11 Jun 2020 10:40:54 -0400 Subject: [PATCH 01/20] WIP test suite automatization --- tests/ipfs.sh | 145 +++++++++++++++++++++++++++++++++++++++++++++++++ tests/local.mk | 75 +++++++++++++------------ 2 files changed, 185 insertions(+), 35 deletions(-) create mode 100644 tests/ipfs.sh diff --git a/tests/ipfs.sh b/tests/ipfs.sh new file mode 100644 index 00000000000..3df3543ed36 --- /dev/null +++ b/tests/ipfs.sh @@ -0,0 +1,145 @@ +source common.sh + +################################################################################ +## Check that the ipfs daemon is enabled in your environment +################################################################################ + +# To see if ipfs is connected to the network, we check if we can see some peers +# other than ourselves. +NPEERS=$(ipfs swarm peers | wc -l) +echo $NPEERS +if (( $NPEERS < 2 )); then + echo "The ipfs daemon doesn't seem to be enabled (can't find peers)" + exit 1 +fi + +################################################################################ +## Create the folders for the source and destination stores +################################################################################ + +IPFS_TESTS=$TEST_ROOT/ipfs_tests + +# Here we define some store locations, one for the initial store we upload, and +# the other three for the destination stores to which we'll copy (one for each +# method) +IPFS_SRC_STORE=$IPFS_TESTS/ipfs_source_store + +IPFS_DST_HTTP_STORE=$IPFS_TESTS/ipfs_dest_http_store +IPFS_DST_HTTP_LOCAL_STORE=$IPFS_TESTS/ipfs_dest_http_local_store +IPFS_DST_IPFS_STORE=$IPFS_TESTS/ipfs_dest_ipfs_store +IPFS_DST_IPNS_STORE=$IPFS_TESTS/ipfs_dest_ipns_store + +################################################################################ +## Generate the keys to sign the store +################################################################################ + +SIGNING_KEY_NAME='nixcache.for.ipfs-1' +SIGNING_KEY_PRI_FILE='nix-cache-key.sec' +SIGNING_KEY_PUB_FILE='nix-cache-key.pub' + +nix-store --generate-binary-cache-key $SIGNING_KEY_NAME $SIGNING_KEY_PRI_FILE $SIGNING_KEY_PUB_FILE + +################################################################################ +## Create, sign and upload the source store +################################################################################ + +mkdir -p $IPFS_SRC_STORE +BUILD_COMMAND='nix-build "" -A hello.src' + +nix copy --to file://$IPFS_SRC_STORE $($BUILD_COMMAND) + +nix sign-paths --store file://$IPFS_SRC_STORE \ + -k ~/nix-cache-key.sec \ + $($BUILD_COMMAND) -r + +IPFS_HASH=$(ipfs add -r $IPFS_SRC_STORE 2>/dev/null | tail -n 1 | awk '{print $2}') + + +# ################################################################################ +# ## Create the http store and download the derivation there +# ################################################################################ + +# mkdir IPFS_DST_HTTP_STORE + +# IPFS_HTTP_PREFIX='https://gateway.ipfs.io/ipfs' + + +# DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ +# --option substituters $IPFS_HTTP_PREFIX/$IPFS_HASH \ +# --store $IPFS_DST_HTTP_STORE \ +# --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) \ +# | tail -2 | head -1 | awk '{print $5}') + +# ### TODO +# ### Check that the download location coincides + +# ################################################################################ +# ## Create the local http store and download the derivation there +# ################################################################################ + +# mkdir IPFS_DST_HTTP_LOCAL_STORE + +# IPFS_HTTP_LOCAL_PREFIX='http://localhost:8080/ipfs' + +# DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ +# --option substituters $IPFS_HTTP_LOCAL_PREFIX/$IPFS_HASH \ +# --store $IPFS_DST_HTTP_LOCAL_STORE \ +# --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) \ +# | tail -2 | head -1 | awk '{print $5}') + +# ### TODO +# ### Check that the download location coincides + +# ################################################################################ +# ## Create the ipfs store and download the derivation there +# ################################################################################ + +# mkdir IPFS_DST_IPFS_STORE + +# IPFS_IPFS_PREFIX='/ipfs' + +# DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ +# --option substituters $IPFS_IPFS_PREFIX/$IPFS_HASH \ +# --store $IPFS_DST_IPFS_STORE \ +# --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) \ +# | tail -2 | head -1 | awk '{print $5}') + +# ### TODO +# ### Check that the download location coincides + +# ################################################################################ +# ## Create the ipns store and download the derivation there +# ################################################################################ + +# # First I have to publish: +# IPNS_ID=$(ipfs name publish $IPFS_HASH | awk '{print substr($3,1,length($3)-1)}') + +# mkdir IPFS_DST_IPNS_STORE +# IPFS_IPNS_PREFIX='/ipns' + +# DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ +# --option substituters $IPFS_IPNS_PREFIX/$IPNS_ID \ +# --store $IPFS_DST_IPFS_STORE \ +# --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) \ +# | tail -2 | head -1 | awk '{print $5}') + +# ### TODO +# ### Check that the download location coincides + +################################################################################ +## Cleanup +################################################################################ + +# Delete the keys used to sign the store +rm $SIGNING_KEY_PRI_FILE $SIGNING_KEY_PUB_FILE + +# Remove all the stores +rm -rf $IPFS_SRC_STORE \ + $IPFS_DST_HTTP_STORE $IPFS_DST_HTTP_LOCAL_STORE \ + $IPFS_DST_IPFS_STORE $IPFS_DST_IPNS_STORE + +# Remove the result of the build +rm result + + +exit 1 diff --git a/tests/local.mk b/tests/local.mk index 536661af88e..e5062b1ba3c 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,38 +1,43 @@ -nix_tests = \ - init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - config.sh \ - gc.sh \ - gc-concurrent.sh \ - gc-auto.sh \ - referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ - gc-runtime.sh check-refs.sh filter-source.sh \ - remote-store.sh export.sh export-graph.sh \ - timeout.sh secure-drv-outputs.sh nix-channel.sh \ - multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ - binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ - check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ - placeholders.sh nix-shell.sh \ - linux-sandbox.sh \ - build-dry.sh \ - build-remote.sh \ - nar-access.sh \ - structured-attrs.sh \ - fetchGit.sh \ - fetchGitRefs.sh \ - fetchGitSubmodules.sh \ - fetchMercurial.sh \ - signing.sh \ - shell.sh \ - brotli.sh \ - pure-eval.sh \ - check.sh \ - plugins.sh \ - search.sh \ - nix-copy-ssh.sh \ - post-hook.sh \ - function-trace.sh \ - recursive.sh - # parallel.sh +nix_tests = ipfs.sh \ +# add.sh + +# nix_tests = add.sh + +# nix_tests = \ +# init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ +# config.sh \ +# gc.sh \ +# gc-concurrent.sh \ +# gc-auto.sh \ +# referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ +# gc-runtime.sh check-refs.sh filter-source.sh \ +# remote-store.sh export.sh export-graph.sh \ +# timeout.sh secure-drv-outputs.sh nix-channel.sh \ +# multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ +# binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ +# check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ +# placeholders.sh nix-shell.sh \ +# linux-sandbox.sh \ +# build-dry.sh \ +# build-remote.sh \ +# nar-access.sh \ +# structured-attrs.sh \ +# fetchGit.sh \ +# fetchGitRefs.sh \ +# fetchGitSubmodules.sh \ +# fetchMercurial.sh \ +# signing.sh \ +# shell.sh \ +# brotli.sh \ +# pure-eval.sh \ +# check.sh \ +# plugins.sh \ +# search.sh \ +# nix-copy-ssh.sh \ +# post-hook.sh \ +# function-trace.sh \ +# recursive.sh +# # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) From 33c61498e973a5fca8a2f8b632f245d69b01f5da Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 11 Jun 2020 14:45:27 -0400 Subject: [PATCH 02/20] With a derivation that doesn't use nixpkgs --- tests/ipfs.sh | 6 ++++-- tests/simple-derivation-builder.sh | 3 +++ tests/simple-derivation.nix | 12 ++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100755 tests/simple-derivation-builder.sh create mode 100644 tests/simple-derivation.nix diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 3df3543ed36..a982c92e261 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -44,9 +44,11 @@ nix-store --generate-binary-cache-key $SIGNING_KEY_NAME $SIGNING_KEY_PRI_FILE $S ################################################################################ mkdir -p $IPFS_SRC_STORE -BUILD_COMMAND='nix-build "" -A hello.src' +# BUILD_COMMAND="nix-build '' -A hello.src" -nix copy --to file://$IPFS_SRC_STORE $($BUILD_COMMAND) +nix copy --to file://$IPFS_SRC_STORE \ + --experimental-features nix-command \ + $(nix-build ./simple-derivation.nix) nix sign-paths --store file://$IPFS_SRC_STORE \ -k ~/nix-cache-key.sec \ diff --git a/tests/simple-derivation-builder.sh b/tests/simple-derivation-builder.sh new file mode 100755 index 00000000000..a47a2d79902 --- /dev/null +++ b/tests/simple-derivation-builder.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +ln -s $input $out diff --git a/tests/simple-derivation.nix b/tests/simple-derivation.nix new file mode 100644 index 00000000000..883e715cf7d --- /dev/null +++ b/tests/simple-derivation.nix @@ -0,0 +1,12 @@ +# This derivation doesn't depend from nixpkgs, so that we can build it during +# our test suite. + +with import ./config.nix; + +mkDerivation { + name = "simple-test-derivation"; + builder = builtins.toFile "builder" "ln -s $input $out"; + # builder = "/home/carlo/code/obsidian/nix/tests/simple-derivation-builder.sh"; + input = + builtins.fetchTarball("http://alpha.gnu.org/gnu/hello/hello-2.6.90.tar.gz"); +} From 25f71d39d9c29f6cb65b7a0e1ed10971b544107f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 11 Jun 2020 15:30:39 -0400 Subject: [PATCH 03/20] need to run init.sh first, sigh --- tests/local.mk | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/local.mk b/tests/local.mk index e5062b1ba3c..a8c45cc73e3 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,5 +1,4 @@ -nix_tests = ipfs.sh \ -# add.sh +nix_tests = init.sh check.sh ipfs.sh # nix_tests = add.sh From 8a3c07326bb5e9ea79c4088729c23ba6ec4e7a21 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 11 Jun 2020 17:18:08 -0400 Subject: [PATCH 04/20] Complete the tests for ipfs --- tests/ipfs.sh | 127 +++++++++++------------------ tests/local.mk | 75 ++++++++--------- tests/simple-derivation-builder.sh | 3 - tests/simple-derivation.nix | 12 --- 4 files changed, 84 insertions(+), 133 deletions(-) delete mode 100755 tests/simple-derivation-builder.sh delete mode 100644 tests/simple-derivation.nix diff --git a/tests/ipfs.sh b/tests/ipfs.sh index a982c92e261..239e1f42f0b 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -1,5 +1,9 @@ source common.sh +# This are for ./fixed.nix +export IMPURE_VAR1=foo +export IMPURE_VAR2=bar + ################################################################################ ## Check that the ipfs daemon is enabled in your environment ################################################################################ @@ -18,6 +22,7 @@ fi ################################################################################ IPFS_TESTS=$TEST_ROOT/ipfs_tests +mkdir $IPFS_TESTS # Here we define some store locations, one for the initial store we upload, and # the other three for the destination stores to which we'll copy (one for each @@ -33,9 +38,9 @@ IPFS_DST_IPNS_STORE=$IPFS_TESTS/ipfs_dest_ipns_store ## Generate the keys to sign the store ################################################################################ -SIGNING_KEY_NAME='nixcache.for.ipfs-1' -SIGNING_KEY_PRI_FILE='nix-cache-key.sec' -SIGNING_KEY_PUB_FILE='nix-cache-key.pub' +SIGNING_KEY_NAME=nixcache.for.ipfs-1 +SIGNING_KEY_PRI_FILE=$IPFS_TESTS/nix-cache-key.sec +SIGNING_KEY_PUB_FILE=$IPFS_TESTS/nix-cache-key.pub nix-store --generate-binary-cache-key $SIGNING_KEY_NAME $SIGNING_KEY_PRI_FILE $SIGNING_KEY_PUB_FILE @@ -44,104 +49,68 @@ nix-store --generate-binary-cache-key $SIGNING_KEY_NAME $SIGNING_KEY_PRI_FILE $S ################################################################################ mkdir -p $IPFS_SRC_STORE -# BUILD_COMMAND="nix-build '' -A hello.src" +# BUILD_COMMAND="nix-build ./dependencies.nix -A input1_drv" +BUILD_COMMAND="nix-build ./fixed.nix -A good" nix copy --to file://$IPFS_SRC_STORE \ - --experimental-features nix-command \ - $(nix-build ./simple-derivation.nix) + $($BUILD_COMMAND) nix sign-paths --store file://$IPFS_SRC_STORE \ - -k ~/nix-cache-key.sec \ + -k $SIGNING_KEY_PRI_FILE \ $($BUILD_COMMAND) -r IPFS_HASH=$(ipfs add -r $IPFS_SRC_STORE 2>/dev/null | tail -n 1 | awk '{print $2}') +################################################################################ +## Create the local http store and download the derivation there +################################################################################ -# ################################################################################ -# ## Create the http store and download the derivation there -# ################################################################################ - -# mkdir IPFS_DST_HTTP_STORE - -# IPFS_HTTP_PREFIX='https://gateway.ipfs.io/ipfs' - - -# DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ -# --option substituters $IPFS_HTTP_PREFIX/$IPFS_HASH \ -# --store $IPFS_DST_HTTP_STORE \ -# --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) \ -# | tail -2 | head -1 | awk '{print $5}') - -# ### TODO -# ### Check that the download location coincides - -# ################################################################################ -# ## Create the local http store and download the derivation there -# ################################################################################ - -# mkdir IPFS_DST_HTTP_LOCAL_STORE - -# IPFS_HTTP_LOCAL_PREFIX='http://localhost:8080/ipfs' - -# DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ -# --option substituters $IPFS_HTTP_LOCAL_PREFIX/$IPFS_HASH \ -# --store $IPFS_DST_HTTP_LOCAL_STORE \ -# --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) \ -# | tail -2 | head -1 | awk '{print $5}') +mkdir $IPFS_DST_HTTP_LOCAL_STORE -# ### TODO -# ### Check that the download location coincides +IPFS_HTTP_LOCAL_PREFIX='http://localhost:8080/ipfs' -# ################################################################################ -# ## Create the ipfs store and download the derivation there -# ################################################################################ +DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ + --option substituters $IPFS_HTTP_LOCAL_PREFIX/$IPFS_HASH \ + --store $IPFS_DST_HTTP_LOCAL_STORE \ + --no-out-link \ + -j0 \ + --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE)) -# mkdir IPFS_DST_IPFS_STORE +################################################################################ +## Create the ipfs store and download the derivation there +################################################################################ -# IPFS_IPFS_PREFIX='/ipfs' +mkdir $IPFS_DST_IPFS_STORE -# DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ -# --option substituters $IPFS_IPFS_PREFIX/$IPFS_HASH \ -# --store $IPFS_DST_IPFS_STORE \ -# --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) \ -# | tail -2 | head -1 | awk '{print $5}') +IPFS_IPFS_PREFIX='/ipfs' -# ### TODO -# ### Check that the download location coincides +DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ + --option substituters $IPFS_IPFS_PREFIX/$IPFS_HASH \ + --store $IPFS_DST_IPFS_STORE \ + --no-out-link \ + -j0 \ + --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE)) -# ################################################################################ -# ## Create the ipns store and download the derivation there -# ################################################################################ -# # First I have to publish: -# IPNS_ID=$(ipfs name publish $IPFS_HASH | awk '{print substr($3,1,length($3)-1)}') +################################################################################ +## Create the ipns store and download the derivation there +################################################################################ -# mkdir IPFS_DST_IPNS_STORE -# IPFS_IPNS_PREFIX='/ipns' +# First I have to publish: +IPNS_ID=$(ipfs name publish $IPFS_HASH | awk '{print substr($3,1,length($3)-1)}') -# DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ -# --option substituters $IPFS_IPNS_PREFIX/$IPNS_ID \ -# --store $IPFS_DST_IPFS_STORE \ -# --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) \ -# | tail -2 | head -1 | awk '{print $5}') +mkdir $IPFS_DST_IPNS_STORE +IPFS_IPNS_PREFIX='/ipns' -# ### TODO -# ### Check that the download location coincides +DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ + --option substituters $IPFS_IPNS_PREFIX/$IPNS_ID \ + --store $IPFS_DST_IPNS_STORE \ + --no-out-link \ + -j0 \ + --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE)) ################################################################################ ## Cleanup ################################################################################ -# Delete the keys used to sign the store -rm $SIGNING_KEY_PRI_FILE $SIGNING_KEY_PUB_FILE - -# Remove all the stores -rm -rf $IPFS_SRC_STORE \ - $IPFS_DST_HTTP_STORE $IPFS_DST_HTTP_LOCAL_STORE \ - $IPFS_DST_IPFS_STORE $IPFS_DST_IPNS_STORE - -# Remove the result of the build -rm result - - -exit 1 +# The cleanup is done automatically by nix/tests/init.sh diff --git a/tests/local.mk b/tests/local.mk index a8c45cc73e3..f8af448d5df 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,42 +1,39 @@ -nix_tests = init.sh check.sh ipfs.sh - -# nix_tests = add.sh - -# nix_tests = \ -# init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ -# config.sh \ -# gc.sh \ -# gc-concurrent.sh \ -# gc-auto.sh \ -# referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ -# gc-runtime.sh check-refs.sh filter-source.sh \ -# remote-store.sh export.sh export-graph.sh \ -# timeout.sh secure-drv-outputs.sh nix-channel.sh \ -# multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ -# binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ -# check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ -# placeholders.sh nix-shell.sh \ -# linux-sandbox.sh \ -# build-dry.sh \ -# build-remote.sh \ -# nar-access.sh \ -# structured-attrs.sh \ -# fetchGit.sh \ -# fetchGitRefs.sh \ -# fetchGitSubmodules.sh \ -# fetchMercurial.sh \ -# signing.sh \ -# shell.sh \ -# brotli.sh \ -# pure-eval.sh \ -# check.sh \ -# plugins.sh \ -# search.sh \ -# nix-copy-ssh.sh \ -# post-hook.sh \ -# function-trace.sh \ -# recursive.sh -# # parallel.sh +nix_tests = \ + init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ + config.sh \ + gc.sh \ + gc-concurrent.sh \ + gc-auto.sh \ + referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ + gc-runtime.sh check-refs.sh filter-source.sh \ + remote-store.sh export.sh export-graph.sh \ + timeout.sh secure-drv-outputs.sh nix-channel.sh \ + multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ + binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ + check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ + placeholders.sh nix-shell.sh \ + linux-sandbox.sh \ + build-dry.sh \ + build-remote.sh \ + nar-access.sh \ + structured-attrs.sh \ + fetchGit.sh \ + fetchGitRefs.sh \ + fetchGitSubmodules.sh \ + fetchMercurial.sh \ + signing.sh \ + shell.sh \ + brotli.sh \ + pure-eval.sh \ + check.sh \ + plugins.sh \ + search.sh \ + nix-copy-ssh.sh \ + post-hook.sh \ + function-trace.sh \ + recursive.sh \ + ipfs.sh + # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/simple-derivation-builder.sh b/tests/simple-derivation-builder.sh deleted file mode 100755 index a47a2d79902..00000000000 --- a/tests/simple-derivation-builder.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -ln -s $input $out diff --git a/tests/simple-derivation.nix b/tests/simple-derivation.nix deleted file mode 100644 index 883e715cf7d..00000000000 --- a/tests/simple-derivation.nix +++ /dev/null @@ -1,12 +0,0 @@ -# This derivation doesn't depend from nixpkgs, so that we can build it during -# our test suite. - -with import ./config.nix; - -mkDerivation { - name = "simple-test-derivation"; - builder = builtins.toFile "builder" "ln -s $input $out"; - # builder = "/home/carlo/code/obsidian/nix/tests/simple-derivation-builder.sh"; - input = - builtins.fetchTarball("http://alpha.gnu.org/gnu/hello/hello-2.6.90.tar.gz"); -} From 0da205bdc097135f0972fe373a7446d51fa1fd5f Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 11 Jun 2020 17:34:57 -0400 Subject: [PATCH 05/20] Update prefixes for ipfs and ipns in test suite --- tests/ipfs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 239e1f42f0b..864519acf64 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -82,10 +82,10 @@ DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ mkdir $IPFS_DST_IPFS_STORE -IPFS_IPFS_PREFIX='/ipfs' +IPFS_IPFS_PREFIX='ipfs://' DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ - --option substituters $IPFS_IPFS_PREFIX/$IPFS_HASH \ + --option substituters $IPFS_IPFS_PREFIX$IPFS_HASH \ --store $IPFS_DST_IPFS_STORE \ --no-out-link \ -j0 \ @@ -100,10 +100,10 @@ DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ IPNS_ID=$(ipfs name publish $IPFS_HASH | awk '{print substr($3,1,length($3)-1)}') mkdir $IPFS_DST_IPNS_STORE -IPFS_IPNS_PREFIX='/ipns' +IPFS_IPNS_PREFIX='ipns://' DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ - --option substituters $IPFS_IPNS_PREFIX/$IPNS_ID \ + --option substituters $IPFS_IPNS_PREFIX$IPNS_ID \ --store $IPFS_DST_IPNS_STORE \ --no-out-link \ -j0 \ From 753ed622af9b704594717b35575a02017e4a5f2c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 11 Jun 2020 21:45:53 +0000 Subject: [PATCH 06/20] Add ipfs as a dep for tests Eventually we should separate test deps. --- release-common.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-common.nix b/release-common.nix index 7e7de005d2f..16832618546 100644 --- a/release-common.nix +++ b/release-common.nix @@ -70,7 +70,8 @@ rec { sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2"; }) ]; */ - })); + })) + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ipfs; propagatedDeps = [ (boehmgc.override { enableLargeConfig = true; }) From 47502232585acde19100cf2c3a9fa47ca49a98a6 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 11 Jun 2020 18:42:58 -0400 Subject: [PATCH 07/20] Restart ipfs when the daemon is not running --- tests/ipfs.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 864519acf64..8dc96354138 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -5,16 +5,20 @@ export IMPURE_VAR1=foo export IMPURE_VAR2=bar ################################################################################ -## Check that the ipfs daemon is enabled in your environment +## Check that the ipfs daemon is present and enabled in your environment ################################################################################ +if [[ -z $(type -p ipfs) ]]; then + echo "Ipfs not installed; skipping ipfs tests" + exit 99 +fi + # To see if ipfs is connected to the network, we check if we can see some peers # other than ourselves. -NPEERS=$(ipfs swarm peers | wc -l) -echo $NPEERS -if (( $NPEERS < 2 )); then - echo "The ipfs daemon doesn't seem to be enabled (can't find peers)" - exit 1 +if (! (ipfs log ls)); +then + echo "Ipfs daemon not detected; initializing.." + ipfs daemon --offline & fi ################################################################################ @@ -97,7 +101,7 @@ DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ ################################################################################ # First I have to publish: -IPNS_ID=$(ipfs name publish $IPFS_HASH | awk '{print substr($3,1,length($3)-1)}') +IPNS_ID=$(ipfs name publish $IPFS_HASH --allow-offline | awk '{print substr($3,1,length($3)-1)}') mkdir $IPFS_DST_IPNS_STORE IPFS_IPNS_PREFIX='ipns://' From b2eccb98f20d17748fe6d64e942f62a6ecdb879d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Jun 2020 15:23:24 +0000 Subject: [PATCH 08/20] Sometimes read permissions dissapear on the test data for some reason --- tests/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/init.sh b/tests/init.sh index c62c4856a93..8b9ded4a446 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -2,7 +2,7 @@ source common.sh test -n "$TEST_ROOT" if test -d "$TEST_ROOT"; then - chmod -R u+w "$TEST_ROOT" + chmod -R u+rw "$TEST_ROOT" rm -rf "$TEST_ROOT" fi mkdir "$TEST_ROOT" From 0068f7d6783da6391d2bac1398a6706e90dfdc79 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Jun 2020 15:23:51 +0000 Subject: [PATCH 09/20] Ensure IPFS daemon is killed after test --- tests/ipfs.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 8dc96354138..bf2886400f9 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -18,7 +18,10 @@ fi if (! (ipfs log ls)); then echo "Ipfs daemon not detected; initializing.." + ipfs init ipfs daemon --offline & + pidIpfsDaemon=$! + trap "kill -9 $pidIpfsDaemon" EXIT fi ################################################################################ @@ -112,9 +115,3 @@ DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ --no-out-link \ -j0 \ --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE)) - -################################################################################ -## Cleanup -################################################################################ - -# The cleanup is done automatically by nix/tests/init.sh From 5352bcbfbc3dfe7bb9d55fb6db392f9f0b2eaf7a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Jun 2020 16:12:20 +0000 Subject: [PATCH 10/20] Just run ipfs tests on linux for now, sigh Not sure what this `nix copy` problem is. I'm pretty sure it's a preexistsing bug. --- release-common.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release-common.nix b/release-common.nix index 16832618546..9c212c9b575 100644 --- a/release-common.nix +++ b/release-common.nix @@ -71,7 +71,9 @@ rec { }) ]; */ })) - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ipfs; + # TODO fix `nix copy` on darwin + # ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ipfs; + # ++ lib.optional stdenv.isLinux ipfs; propagatedDeps = [ (boehmgc.override { enableLargeConfig = true; }) From 7599cf040d0a3f03a593169fd7f48030e739e476 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Jun 2020 16:13:52 +0000 Subject: [PATCH 11/20] Fix syntax error from bad paste --- release-common.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-common.nix b/release-common.nix index 9c212c9b575..e9e85956af4 100644 --- a/release-common.nix +++ b/release-common.nix @@ -73,7 +73,7 @@ rec { })) # TODO fix `nix copy` on darwin # ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ipfs; - # ++ lib.optional stdenv.isLinux ipfs; + ++ lib.optional stdenv.isLinux ipfs; propagatedDeps = [ (boehmgc.override { enableLargeConfig = true; }) From 4348615044e0058114e81a4ea7e588725aca9a7c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Jun 2020 19:49:32 +0000 Subject: [PATCH 12/20] Remove `NIX_REMOTE=local` from ipfs test The test infra will handle NIX_REMOTE --- tests/ipfs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index bf2886400f9..3fb08bf49a9 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -76,7 +76,7 @@ mkdir $IPFS_DST_HTTP_LOCAL_STORE IPFS_HTTP_LOCAL_PREFIX='http://localhost:8080/ipfs' -DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ +DOWNLOAD_LOCATION=$($BUILD_COMMAND \ --option substituters $IPFS_HTTP_LOCAL_PREFIX/$IPFS_HASH \ --store $IPFS_DST_HTTP_LOCAL_STORE \ --no-out-link \ @@ -91,7 +91,7 @@ mkdir $IPFS_DST_IPFS_STORE IPFS_IPFS_PREFIX='ipfs://' -DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ +DOWNLOAD_LOCATION=$($BUILD_COMMAND \ --option substituters $IPFS_IPFS_PREFIX$IPFS_HASH \ --store $IPFS_DST_IPFS_STORE \ --no-out-link \ @@ -109,7 +109,7 @@ IPNS_ID=$(ipfs name publish $IPFS_HASH --allow-offline | awk '{print substr($3,1 mkdir $IPFS_DST_IPNS_STORE IPFS_IPNS_PREFIX='ipns://' -DOWNLOAD_LOCATION=$(NIX_REMOTE=local $BUILD_COMMAND \ +DOWNLOAD_LOCATION=$($BUILD_COMMAND \ --option substituters $IPFS_IPNS_PREFIX$IPNS_ID \ --store $IPFS_DST_IPNS_STORE \ --no-out-link \ From 67c63b954cccbdbaf17947b3d57f79a55d9f98c5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 15:51:39 -0500 Subject: [PATCH 13/20] Only call build once in tests/ipfs.sh --- tests/ipfs.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 3fb08bf49a9..5fd97741fa6 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -56,15 +56,12 @@ nix-store --generate-binary-cache-key $SIGNING_KEY_NAME $SIGNING_KEY_PRI_FILE $S ################################################################################ mkdir -p $IPFS_SRC_STORE -# BUILD_COMMAND="nix-build ./dependencies.nix -A input1_drv" -BUILD_COMMAND="nix-build ./fixed.nix -A good" +storePaths=$(nix-build ./fixed.nix -A good) -nix copy --to file://$IPFS_SRC_STORE \ - $($BUILD_COMMAND) +nix copy --to file://$IPFS_SRC_STORE $storePaths nix sign-paths --store file://$IPFS_SRC_STORE \ - -k $SIGNING_KEY_PRI_FILE \ - $($BUILD_COMMAND) -r + -k $SIGNING_KEY_PRI_FILE storePaths IPFS_HASH=$(ipfs add -r $IPFS_SRC_STORE 2>/dev/null | tail -n 1 | awk '{print $2}') From e1d00917ab4a1ad38da95636849c356bfad2b224 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 15:52:01 -0500 Subject: [PATCH 14/20] Clear store when beginning ipfs test --- tests/ipfs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 5fd97741fa6..307958b4df6 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -24,6 +24,8 @@ then trap "kill -9 $pidIpfsDaemon" EXIT fi +clearStore + ################################################################################ ## Create the folders for the source and destination stores ################################################################################ From 4f3a4c9e354647e26907a3e99ccdded35015ccb5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 16:44:16 -0500 Subject: [PATCH 15/20] Fix typo in ipfs.sh --- tests/ipfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 307958b4df6..adeef876cad 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -63,7 +63,7 @@ storePaths=$(nix-build ./fixed.nix -A good) nix copy --to file://$IPFS_SRC_STORE $storePaths nix sign-paths --store file://$IPFS_SRC_STORE \ - -k $SIGNING_KEY_PRI_FILE storePaths + -k $SIGNING_KEY_PRI_FILE $storePaths IPFS_HASH=$(ipfs add -r $IPFS_SRC_STORE 2>/dev/null | tail -n 1 | awk '{print $2}') From 4b83835502b99764f7200eac4d7c427366238391 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 17:52:25 -0500 Subject: [PATCH 16/20] Fixup unset BUILD_COMMAND --- tests/ipfs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index adeef876cad..4841a7ccc1a 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -75,7 +75,7 @@ mkdir $IPFS_DST_HTTP_LOCAL_STORE IPFS_HTTP_LOCAL_PREFIX='http://localhost:8080/ipfs' -DOWNLOAD_LOCATION=$($BUILD_COMMAND \ +DOWNLOAD_LOCATION=$(nix-build ./fixed.nix -A good \ --option substituters $IPFS_HTTP_LOCAL_PREFIX/$IPFS_HASH \ --store $IPFS_DST_HTTP_LOCAL_STORE \ --no-out-link \ @@ -90,7 +90,7 @@ mkdir $IPFS_DST_IPFS_STORE IPFS_IPFS_PREFIX='ipfs://' -DOWNLOAD_LOCATION=$($BUILD_COMMAND \ +DOWNLOAD_LOCATION=$(nix-build ./fixed.nix -A good \ --option substituters $IPFS_IPFS_PREFIX$IPFS_HASH \ --store $IPFS_DST_IPFS_STORE \ --no-out-link \ @@ -108,7 +108,7 @@ IPNS_ID=$(ipfs name publish $IPFS_HASH --allow-offline | awk '{print substr($3,1 mkdir $IPFS_DST_IPNS_STORE IPFS_IPNS_PREFIX='ipns://' -DOWNLOAD_LOCATION=$($BUILD_COMMAND \ +DOWNLOAD_LOCATION=$(nix-build ./fixed.nix -A good \ --option substituters $IPFS_IPNS_PREFIX$IPNS_ID \ --store $IPFS_DST_IPNS_STORE \ --no-out-link \ From 3cb4a0baadb0e3f0616b1fc8367452334dbf20bf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 13 Jun 2020 00:29:35 -0500 Subject: [PATCH 17/20] Try to make /nar before nix copy in tests/ipfs.sh --- tests/ipfs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 4841a7ccc1a..01765b5446d 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -58,6 +58,7 @@ nix-store --generate-binary-cache-key $SIGNING_KEY_NAME $SIGNING_KEY_PRI_FILE $S ################################################################################ mkdir -p $IPFS_SRC_STORE +mkdir $IPFS_SRC_STORE/nar storePaths=$(nix-build ./fixed.nix -A good) nix copy --to file://$IPFS_SRC_STORE $storePaths From ad96d2111e4dd8a9011bb66d790c964df6c8d296 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 Jun 2020 14:37:07 +0000 Subject: [PATCH 18/20] Revert "Just run ipfs tests on linux for now, sigh" This reverts commit 5352bcbfbc3dfe7bb9d55fb6db392f9f0b2eaf7a. --- release-common.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/release-common.nix b/release-common.nix index e9e85956af4..16832618546 100644 --- a/release-common.nix +++ b/release-common.nix @@ -71,9 +71,7 @@ rec { }) ]; */ })) - # TODO fix `nix copy` on darwin - # ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ipfs; - ++ lib.optional stdenv.isLinux ipfs; + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ipfs; propagatedDeps = [ (boehmgc.override { enableLargeConfig = true; }) From 08ee3665210d1958e6c3bd7244c7a7078d9d94de Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 Jun 2020 15:44:57 +0000 Subject: [PATCH 19/20] ipfs test: Inline some vars used once --- tests/ipfs.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 01765b5446d..257fa81a7c2 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -89,10 +89,8 @@ DOWNLOAD_LOCATION=$(nix-build ./fixed.nix -A good \ mkdir $IPFS_DST_IPFS_STORE -IPFS_IPFS_PREFIX='ipfs://' - DOWNLOAD_LOCATION=$(nix-build ./fixed.nix -A good \ - --option substituters $IPFS_IPFS_PREFIX$IPFS_HASH \ + --option substituters 'ipfs://'$IPFS_HASH \ --store $IPFS_DST_IPFS_STORE \ --no-out-link \ -j0 \ @@ -107,10 +105,9 @@ DOWNLOAD_LOCATION=$(nix-build ./fixed.nix -A good \ IPNS_ID=$(ipfs name publish $IPFS_HASH --allow-offline | awk '{print substr($3,1,length($3)-1)}') mkdir $IPFS_DST_IPNS_STORE -IPFS_IPNS_PREFIX='ipns://' DOWNLOAD_LOCATION=$(nix-build ./fixed.nix -A good \ - --option substituters $IPFS_IPNS_PREFIX$IPNS_ID \ + --option substituters 'ipns://'$IPNS_ID \ --store $IPFS_DST_IPNS_STORE \ --no-out-link \ -j0 \ From 953ceba4cd6a38c439dc737eee525a1fc7ffc5d6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 Jun 2020 12:53:41 -0400 Subject: [PATCH 20/20] Identify real issue (#3695) and hack around it --- tests/ipfs.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 257fa81a7c2..9cad1c381f6 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -58,10 +58,13 @@ nix-store --generate-binary-cache-key $SIGNING_KEY_NAME $SIGNING_KEY_PRI_FILE $S ################################################################################ mkdir -p $IPFS_SRC_STORE -mkdir $IPFS_SRC_STORE/nar -storePaths=$(nix-build ./fixed.nix -A good) +storePaths=$(nix-build ./fixed.nix -A good | sort | uniq) -nix copy --to file://$IPFS_SRC_STORE $storePaths +# Hack around https://github.com/NixOS/nix/issues/3695 +for path in $storePaths; do + nix copy --to file://$IPFS_SRC_STORE $path +done +unset path nix sign-paths --store file://$IPFS_SRC_STORE \ -k $SIGNING_KEY_PRI_FILE $storePaths