From 50d578504eed8539e749e73e8cf02ffcdf2e3381 Mon Sep 17 00:00:00 2001 From: Lukas Vogel Date: Mon, 30 Dec 2019 15:32:06 +0100 Subject: [PATCH 1/4] CI: add revocation tests to pipeline 2 Also: - use accept artifacts for diff tests in check generated tasks, this dir always exists also on the local CI. - generate acceptance steps for bazelified acceptance tests. --- .buildkite/pipeline2.sh | 21 +++++++++++++ .buildkite/pipeline_buildlint.yml | 49 ++++++++++++------------------- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/.buildkite/pipeline2.sh b/.buildkite/pipeline2.sh index 185e12c9f7..bbdf61e314 100755 --- a/.buildkite/pipeline2.sh +++ b/.buildkite/pipeline2.sh @@ -23,5 +23,26 @@ gen_acceptance() { done } +# gen_acceptance2 generates steps for bazel tests in acceptance folder. +gen_acceptance2() { + for test in $(bazel query 'kind(sh_test, //acceptance/...)' 2>/dev/null); do + # test has the format //acceptance/:_test + name=$(echo $test | cut -d ':' -f 1) + name=${name#'//acceptance/'} + echo " - label: \"Acceptance: $name\"" + echo " command:" + echo " - mkdir -p \$\$ACCEPTANCE_ARTIFACTS" + echo " - bazel test --action_env=ACCEPTANCE_ARTIFACTS $test" + echo " key: ${name}_acceptance" + echo " artifact_paths:" + echo " - \"artifacts.out/**/*\"" + echo " retry:" + echo " automatic:" + echo " - exit_status: -1 # Agent was lost" + echo " - exit_status: 255 # Forced agent shutdown" + done +} + cat .buildkite/pipeline_buildlint.yml +gen_acceptance2 gen_acceptance diff --git a/.buildkite/pipeline_buildlint.yml b/.buildkite/pipeline_buildlint.yml index 4fdce619a0..0521e4b14c 100644 --- a/.buildkite/pipeline_buildlint.yml +++ b/.buildkite/pipeline_buildlint.yml @@ -21,14 +21,12 @@ steps: timeout_in_minutes: 10 - label: "Check generated go_deps.bzl file is up to date with go.mod" command: - - "rm -rf /tmp/$BUILDKITE_STEP_ID/" - - "mkdir -p /tmp/$BUILDKITE_STEP_ID/" - - "cp go.mod go.sum go_deps.bzl /tmp/$BUILDKITE_STEP_ID/" - - "make godeps -B" - - "bazel-${BUILDKITE_PIPELINE_SLUG}/external/go_sdk/bin/go mod tidy" - - "diff -u /tmp/$BUILDKITE_STEP_ID/go.mod go.mod" - - "diff -u /tmp/$BUILDKITE_STEP_ID/go.sum go.sum" - - "diff -u /tmp/$BUILDKITE_STEP_ID/go_deps.bzl go_deps.bzl" + - "cp go.mod go.sum go_deps.bzl $$ACCEPTANCE_ARTIFACTS/" + - make godeps -B + - "bazel-${USER}/external/go_sdk/bin/go mod tidy" + - "diff -u $$ACCEPTANCE_ARTIFACTS/go.mod go.mod" + - "diff -u $$ACCEPTANCE_ARTIFACTS/go.sum go.sum" + - "diff -u $$ACCEPTANCE_ARTIFACTS/go_deps.bzl go_deps.bzl" key: go_deps_lint retry: automatic: @@ -36,11 +34,9 @@ steps: - exit_status: 255 # Forced agent shutdown - label: "Check generated go/proto files in git" command: - - "rm -rf /tmp/$BUILDKITE_STEP_ID/" - - "mkdir -p /tmp/$BUILDKITE_STEP_ID/" - - "cp -R go/proto/ /tmp/$BUILDKITE_STEP_ID/" - - "make gogen" - - "diff -ur /tmp/$BUILDKITE_STEP_ID/proto/ go/proto/" + - "cp -R go/proto/ $$ACCEPTANCE_ARTIFACTS" + - make gogen + - "diff -ur $$ACCEPTANCE_ARTIFACTS/proto/ go/proto/" key: go_gen_lint retry: automatic: @@ -53,26 +49,17 @@ steps: automatic: - exit_status: -1 # Agent was lost - exit_status: 255 # Forced agent shutdown - - label: "Acceptance: sig_failover" + - label: "Revocation tests" command: - - mkdir -p $$ACCEPTANCE_ARTIFACTS - - bazel test --action_env=ACCEPTANCE_ARTIFACTS //acceptance/sig_failover:sig_failover_test - key: sig_failover + - bazel --bazelrc=.bazelrc_ci build //:scion //:scion-ci >/dev/null 2>&1 + - tar -kxf bazel-bin/scion.tar -C bin --overwrite + - tar -kxf bazel-bin/scion-ci.tar -C bin --overwrite + - ./scion.sh topology nobuild + - ./scion.sh run nobuild && sleep 10 + - ./bin/end2end_integration -log.console warn + - ./integration/revocation_test.sh + key: revocation_tests retry: automatic: - exit_status: -1 # Agent was lost - exit_status: 255 # Forced agent shutdown - artifact_paths: - - "artifacts.out/**/*" - - label: "Acceptance: sig_short_exp_time" - command: - - mkdir -p $$ACCEPTANCE_ARTIFACTS - - bazel test --action_env=ACCEPTANCE_ARTIFACTS //acceptance/sig_short_exp_time:sig_short_exp_time_test - key: sig_short_exp_time - retry: - automatic: - - exit_status: -1 # Agent was lost - - exit_status: 255 # Forced agent shutdown - artifact_paths: - - "artifacts.out/**/*" - From 2eb6e81cc5bb4e110899cf1658cf2a3d48b869ef Mon Sep 17 00:00:00 2001 From: Lukas Vogel Date: Tue, 31 Dec 2019 08:53:41 +0100 Subject: [PATCH 2/4] Pre-gen accept_artifacts folder & collect artifacts --- .buildkite/hooks/pre-command | 1 + .buildkite/pipeline2.sh | 2 -- .buildkite/pipeline_buildlint.yml | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 5ec198f711..9e428f9ae4 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -8,6 +8,7 @@ echo "Clean existing environment" if [ "$BUILDKITE_PIPELINE_SLUG" == "scionproto2" ]; then export ACCEPTANCE_ARTIFACTS="$PWD/accept_artifacts" + mkdir -p "$ACCEPTANCE_ARTIFACTS" fi echo "Starting bazel remote cache proxy" diff --git a/.buildkite/pipeline2.sh b/.buildkite/pipeline2.sh index bbdf61e314..390374e1a6 100755 --- a/.buildkite/pipeline2.sh +++ b/.buildkite/pipeline2.sh @@ -8,7 +8,6 @@ gen_acceptance() { name="$(basename ${test%_acceptance})" echo " - label: \"Acceptance: $name\"" echo " command:" - echo " - \"mkdir -p \$\$ACCEPTANCE_ARTIFACTS\"" echo " - ./acceptance/ctl gsetup" echo " - ./acceptance/ctl grun $name" echo " key: ${name}_acceptance" @@ -31,7 +30,6 @@ gen_acceptance2() { name=${name#'//acceptance/'} echo " - label: \"Acceptance: $name\"" echo " command:" - echo " - mkdir -p \$\$ACCEPTANCE_ARTIFACTS" echo " - bazel test --action_env=ACCEPTANCE_ARTIFACTS $test" echo " key: ${name}_acceptance" echo " artifact_paths:" diff --git a/.buildkite/pipeline_buildlint.yml b/.buildkite/pipeline_buildlint.yml index 0521e4b14c..250c096815 100644 --- a/.buildkite/pipeline_buildlint.yml +++ b/.buildkite/pipeline_buildlint.yml @@ -23,7 +23,7 @@ steps: command: - "cp go.mod go.sum go_deps.bzl $$ACCEPTANCE_ARTIFACTS/" - make godeps -B - - "bazel-${USER}/external/go_sdk/bin/go mod tidy" + - "bazel-${BUILDKITE_PIPELINE_SLUG}/external/go_sdk/bin/go mod tidy" - "diff -u $$ACCEPTANCE_ARTIFACTS/go.mod go.mod" - "diff -u $$ACCEPTANCE_ARTIFACTS/go.sum go.sum" - "diff -u $$ACCEPTANCE_ARTIFACTS/go_deps.bzl go_deps.bzl" @@ -59,6 +59,8 @@ steps: - ./bin/end2end_integration -log.console warn - ./integration/revocation_test.sh key: revocation_tests + artifact_paths: + - "artifacts.out/**/*" retry: automatic: - exit_status: -1 # Agent was lost From a430f63315accf59c99c6274e5421d83780ca5b6 Mon Sep 17 00:00:00 2001 From: Lukas Vogel Date: Tue, 31 Dec 2019 10:52:47 +0100 Subject: [PATCH 3/4] improvements --- .buildkite/hooks/pre-artifact | 2 +- .buildkite/hooks/pre-command | 17 +++++++++++++++-- .buildkite/hooks/pre-exit | 2 +- .buildkite/pipeline2.sh | 8 ++++---- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.buildkite/hooks/pre-artifact b/.buildkite/hooks/pre-artifact index 251cff4002..20a523a4ea 100755 --- a/.buildkite/hooks/pre-artifact +++ b/.buildkite/hooks/pre-artifact @@ -45,6 +45,6 @@ save "logs" save "traces" save "gen" save "gen-cache" -save "accept_artifacts" +save "$TEST_ARTIFACTS" tar chaf "artifacts.out/$ARTIFACTS.tar.gz" -C "$ARTIFACTS_DIR" "$ARTIFACTS" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 9e428f9ae4..b0e7a636a2 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -4,11 +4,24 @@ set -euo pipefail echo "Clean existing environment" +# TEST_ARTIFACTS is a generic folder for artifacts that need to be used during +# testing. It will be collected in the artifacts in the end. +# Note that this has to be defined in a hook, because in the pipeline the $PWD +# would be evaluated to early (at pipeline expansion) and thus it would possibly +# be the wrong value on the agent a step is actually executed. +export TEST_ARTIFACTS="$PWD/test_artifacts" + . .buildkite/hooks/pre-exit +# Make sure the test artifacts folder exitsts. +echo "Create test artifacts: mkdir -p $TEST_ARTIFACTS" +mkdir -p "$TEST_ARTIFACTS" + +# Conditionally export ACCEPTANCE_ARTIFACTS it is needed for acceptance tests. +# We only export it for the new pipeline because the old pipeline has its own +# definition. if [ "$BUILDKITE_PIPELINE_SLUG" == "scionproto2" ]; then - export ACCEPTANCE_ARTIFACTS="$PWD/accept_artifacts" - mkdir -p "$ACCEPTANCE_ARTIFACTS" + export ACCEPTANCE_ARTIFACTS="$TEST_ARTIFACTS" fi echo "Starting bazel remote cache proxy" diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index 650e3cc822..2a92798fc0 100644 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -10,4 +10,4 @@ docker network prune -f echo "Remove leftover volumes" docker volume prune -f -rm -rf bazel-testlogs logs/* traces gen gen-cache accept_artifacts +rm -rf bazel-testlogs logs/* traces gen gen-cache "$TEST_ARTIFACTS" diff --git a/.buildkite/pipeline2.sh b/.buildkite/pipeline2.sh index 390374e1a6..6d07c40091 100755 --- a/.buildkite/pipeline2.sh +++ b/.buildkite/pipeline2.sh @@ -22,15 +22,15 @@ gen_acceptance() { done } -# gen_acceptance2 generates steps for bazel tests in acceptance folder. -gen_acceptance2() { +# gen_bazel_acceptance generates steps for bazel tests in acceptance folder. +gen_bazel_acceptance() { for test in $(bazel query 'kind(sh_test, //acceptance/...)' 2>/dev/null); do # test has the format //acceptance/:_test name=$(echo $test | cut -d ':' -f 1) name=${name#'//acceptance/'} echo " - label: \"Acceptance: $name\"" echo " command:" - echo " - bazel test --action_env=ACCEPTANCE_ARTIFACTS $test" + echo " - bazel test $test" echo " key: ${name}_acceptance" echo " artifact_paths:" echo " - \"artifacts.out/**/*\"" @@ -42,5 +42,5 @@ gen_acceptance2() { } cat .buildkite/pipeline_buildlint.yml -gen_acceptance2 +gen_bazel_acceptance gen_acceptance From 372f58416c48c4d4b153163f4fca6624127f5a8a Mon Sep 17 00:00:00 2001 From: Lukas Vogel Date: Tue, 31 Dec 2019 11:12:06 +0100 Subject: [PATCH 4/4] fix refs --- .buildkite/pipeline_buildlint.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline_buildlint.yml b/.buildkite/pipeline_buildlint.yml index 250c096815..48a9f2cc87 100644 --- a/.buildkite/pipeline_buildlint.yml +++ b/.buildkite/pipeline_buildlint.yml @@ -21,12 +21,12 @@ steps: timeout_in_minutes: 10 - label: "Check generated go_deps.bzl file is up to date with go.mod" command: - - "cp go.mod go.sum go_deps.bzl $$ACCEPTANCE_ARTIFACTS/" + - "cp go.mod go.sum go_deps.bzl $$TEST_ARTIFACTS/" - make godeps -B - "bazel-${BUILDKITE_PIPELINE_SLUG}/external/go_sdk/bin/go mod tidy" - - "diff -u $$ACCEPTANCE_ARTIFACTS/go.mod go.mod" - - "diff -u $$ACCEPTANCE_ARTIFACTS/go.sum go.sum" - - "diff -u $$ACCEPTANCE_ARTIFACTS/go_deps.bzl go_deps.bzl" + - "diff -u $$TEST_ARTIFACTS/go.mod go.mod" + - "diff -u $$TEST_ARTIFACTS/go.sum go.sum" + - "diff -u $$TEST_ARTIFACTS/go_deps.bzl go_deps.bzl" key: go_deps_lint retry: automatic: @@ -34,9 +34,9 @@ steps: - exit_status: 255 # Forced agent shutdown - label: "Check generated go/proto files in git" command: - - "cp -R go/proto/ $$ACCEPTANCE_ARTIFACTS" + - "cp -R go/proto/ $$TEST_ARTIFACTS" - make gogen - - "diff -ur $$ACCEPTANCE_ARTIFACTS/proto/ go/proto/" + - "diff -ur $$TEST_ARTIFACTS/proto/ go/proto/" key: go_gen_lint retry: automatic: