From 755a4e4a4e83c265c5b0fd8604992abd3bafea8d Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Thu, 24 Jun 2021 13:24:30 -0400 Subject: [PATCH 01/19] Support running the pipeline upload script locally --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 8678e3f2e87..1ed9da76c0a 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -10,7 +10,7 @@ BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' [[ -z "$ROUNDS" ]] && export ROUNDS='1' # attach pipeline documentation export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" -export RETRY="$(buildkite-agent meta-data get pipeline-upload-retries --default '0')" +export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' From dcfb1e6b9288e5e7a9e223a11c1aab49440a10d0 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Thu, 24 Jun 2021 13:30:30 -0400 Subject: [PATCH 02/19] Fix step retry count --- .cicd/generate-pipeline.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 1ed9da76c0a..0268a5aa327 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -15,6 +15,7 @@ if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' fi +[[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) From effca93e42cdcc2b9cf4e3d6e8e5c41e0b33f354 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 16:33:12 -0400 Subject: [PATCH 03/19] Alphabetize test steps --- .cicd/generate-pipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 0268a5aa327..598da203083 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -309,7 +309,7 @@ EOF echo ' # serial tests' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do IFS=$oIFS - SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }')" + SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }' | sort | uniq)" for TEST_NAME in $SERIAL_TESTS; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat < Date: Fri, 25 Jun 2021 17:42:18 -0400 Subject: [PATCH 04/19] Only create serial test steps when TEST is non-empty --- .cicd/generate-pipeline.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 598da203083..4354c815242 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -194,7 +194,7 @@ if [[ "$DCMAKE_BUILD_TYPE" != 'Debug' ]]; then echo " # round $ROUND of $ROUNDS" # parallel tests echo ' # parallel tests' - echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do + [[ -z "$TEST" ]] && echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat < Date: Fri, 25 Jun 2021 17:45:12 -0400 Subject: [PATCH 05/19] Support PCRE test filter --- .cicd/generate-pipeline.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 4354c815242..c738e05670e 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -309,7 +309,11 @@ EOF echo ' # serial tests' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do IFS=$oIFS - SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }' | sort | uniq)" + if [[ -z "$TEST" ]]; then + SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq)" + else + SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq | grep -P "^$TEST$")" + fi for TEST_NAME in $SERIAL_TESTS; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat < Date: Fri, 25 Jun 2021 16:43:00 -0400 Subject: [PATCH 06/19] Support ROUND_SIZE --- .cicd/generate-pipeline.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index c738e05670e..de4b29d4a7f 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -5,9 +5,10 @@ set -eo pipefail export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::brew::buildkite-agent'} export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.6_6C_14G_80G'} export PLATFORMS_JSON_ARRAY='[]' +[[ -z "$ROUNDS" ]] && export ROUNDS='1' +[[ -z "$ROUND_SIZE" ]] && export ROUND_SIZE='5' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' -[[ -z "$ROUNDS" ]] && export ROUNDS='1' # attach pipeline documentation export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" @@ -312,7 +313,7 @@ EOF if [[ -z "$TEST" ]]; then SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq)" else - SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq | grep -P "^$TEST$")" + SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq | grep -P "^$TEST$" | awk "{while(i++<$ROUND_SIZE)print;i=0}")" fi for TEST_NAME in $SERIAL_TESTS; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then From abbbd1b7546e64fe851cfd6473f0593f16e788f3 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 17:52:49 -0400 Subject: [PATCH 07/19] Consistent TIMEOUT formatting --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index de4b29d4a7f..d76b93f4855 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -683,7 +683,7 @@ fi buildkite-agent artifact upload eosio.rb agents: queue: "automation-basic-builder-fleet" - timeout: "${TIMEOUT:-5}" + timeout: ${TIMEOUT:-5} skip: ${SKIP_PACKAGE_BUILDER}${SKIP_MAC}${SKIP_MACOS_10_14} - label: ":docker: :ubuntu: Docker - Build 18.04 Docker Image" From 25beda9ad81cb4da024b1f302d73f2f2986b6967 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 18:01:23 -0400 Subject: [PATCH 08/19] Put default ROUND_SIZE back to 1 because it is used for all builds now --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index d76b93f4855..07b1f148897 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -6,7 +6,7 @@ export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas:: export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.6_6C_14G_80G'} export PLATFORMS_JSON_ARRAY='[]' [[ -z "$ROUNDS" ]] && export ROUNDS='1' -[[ -z "$ROUND_SIZE" ]] && export ROUND_SIZE='5' +[[ -z "$ROUND_SIZE" ]] && export ROUND_SIZE='1' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation From 5d2c86d362f69c62c4fbdb32b31908a778b10705 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 18:07:02 -0400 Subject: [PATCH 09/19] Remove extraneous newlines --- .cicd/generate-pipeline.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 07b1f148897..b8a5aea0497 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -249,7 +249,6 @@ EOF EOF fi - echo done # wasm spec tests echo ' # wasm spec tests' @@ -304,7 +303,6 @@ EOF EOF fi - echo done # serial tests echo ' # serial tests' @@ -366,7 +364,6 @@ EOF EOF fi - echo done IFS=$nIFS done @@ -426,7 +423,6 @@ EOF EOF fi - echo done IFS=$nIFS done From 6f968d34823945c173b6df860b72d4ee56a2e76c Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 18:07:45 -0400 Subject: [PATCH 10/19] Exclude docker build and install step when TEST is non-empty, too --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index b8a5aea0497..0f9f179a816 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -172,7 +172,7 @@ EOF EOF fi done -cat < Date: Fri, 25 Jun 2021 19:10:00 -0400 Subject: [PATCH 11/19] Add guard against users spawning too many jobs by accident --- .cicd/generate-pipeline.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 0f9f179a816..9ed5a793410 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -17,6 +17,19 @@ if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" +# guard against accidentally spawning too many jobs +if (( $ROUNDS > 1 || $ROUND_SIZE > 1 )) && [[ -z "$TEST" ]]; then + echo '+++ :no_entry: WARNING: Your parameters will spawn a very large number of jobs!' 1>&2 + echo "Setting ROUNDS='$ROUNDS' and/or ROUND_SIZE='$ROUND_SIZE' in the environment without also setting TEST to a specific test will cause ALL tests to be run $(( $ROUNDS * $ROUND_SIZE )) times, which will consume a large number of agents! We recommend doing stability testing on ONE test at a time. If you're certain you want to do this, set TEST='.*' to run all tests $(( $ROUNDS * $ROUND_SIZE )) times." 1>&2 + [[ "$BUILDKITE" == 'true' ]] && cat | buildkite-agent annotate --append --style 'error' --context 'no-TEST' <<-MD + Your build was cancelled because you set \`ROUNDS\` and/or \`ROUND_SIZE\` without also setting \`TEST\` in your build environment. This would cause each test to be run $(( $ROUNDS * $ROUND_SIZE )) times, which will consume a lot of Buildkite agents. + + We recommend stability testing one test at a time by setting \`TEST\` equal to the test name. Alternatively, you can specify a set of test names using [Perl-Compatible Regular Expressions](https://www.debuggex.com/cheatsheet/regex/pcre), where \`TEST\` is parsed as \`^${TEST}$\`. + + If you _really_ meant to run every test $(( $ROUNDS * $ROUND_SIZE )) times, set \`TEST='.*'\`. +MD + exit 255 +fi # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) From 2cbae14e8e6cbc4e28dce2f06e6135f3785242f6 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 19:56:06 -0400 Subject: [PATCH 12/19] If they want to run every test, just spawn the jobs like normal --- .cicd/generate-pipeline.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 9ed5a793410..6a2e7dcda28 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -29,6 +29,8 @@ if (( $ROUNDS > 1 || $ROUND_SIZE > 1 )) && [[ -z "$TEST" ]]; then If you _really_ meant to run every test $(( $ROUNDS * $ROUND_SIZE )) times, set \`TEST='.*'\`. MD exit 255 +elif [[ "$TEST" = '.*' ]]; then # if they want to run every test, just spawn the jobs like normal + unset TEST fi # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then From 808c7531c935d4a82786067301a661acccab6cbd Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 21:32:05 -0400 Subject: [PATCH 13/19] Fix link to current documentation --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 6a2e7dcda28..43fd1b9cd6b 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -10,7 +10,7 @@ export PLATFORMS_JSON_ARRAY='[]' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation -export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" +export DOCS_URL="https://github.com/EOSIO/eos/blob/$(git rev-parse HEAD)/.cicd/README.md" export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' From 4a45d59b42b8e13c4a16d3dfc0f26d9140be4818 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 21:37:24 -0400 Subject: [PATCH 14/19] Update documentation --- .cicd/README.md | 2 + .cicd/eosio-test-stability.md | 80 +++++++++++++++++++++++++++++++++++ .cicd/generate-pipeline.sh | 1 + 3 files changed, 83 insertions(+) create mode 100644 .cicd/eosio-test-stability.md diff --git a/.cicd/README.md b/.cicd/README.md index 93ad073ae53..d2480f17427 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -94,12 +94,14 @@ Pipeline | Details [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | runs tests that need more time on merge commits [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | loads the current version of `nodeos` from state files generated by specific previous versions of `nodeos` in each [eosio](https://buildkite.com/EOSIO/eosio) build ([Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md)) [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | sync the current version of `nodeos` past genesis from peers on common public chains as a smoke test, for each [eosio](https://buildkite.com/EOSIO/eosio) build +[eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) | prove or disprove test stability by running a test thousands of times ## See Also - Buildkite - [DevDocs](https://github.com/EOSIO/devdocs/wiki/Buildkite) - [eosio-resume-from-state Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) + - [Stability Testing](https://github.com/EOSIO/eos/blob/HEAD/.cicd/eosio-test-stability.md) - [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel diff --git a/.cicd/eosio-test-stability.md b/.cicd/eosio-test-stability.md new file mode 100644 index 00000000000..4000c87a406 --- /dev/null +++ b/.cicd/eosio-test-stability.md @@ -0,0 +1,80 @@ +# Stability Testing +Stability testing of EOSIO unit and integration tests is done in the [eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) pipeline. It will take thousands of runs of any given test to identify it as "stable" or "unstable". Runs should be split evenly across "pinned" (fixed dependency version) and "unpinned" (default dependency version) builds because, sometimes, test instability is only expressed in one of these environments. Finally, stability testing should be performed on the Linux fleet first because this fleet is effectively infinite. Once stability is demonstrated on Linux, testing can be performed on the finite macOS Anka fleet. + +
+See More + +## Index +1. [Configuration](eosio-test-stability.md#configuration) + 1. [Variables](eosio-test-stability.md#variables) + 1. [Runs](eosio-test-stability.md#runs) + 1. [Examples](eosio-test-stability.md#examples) +1. [See Also](eosio-test-stability.md#see-also) + +## Configuration +The [eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) pipeline uses the same pipeline upload script as [eosio](https://buildkite.com/EOSIO/eosio), [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned), and [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt), so all variables from the [pipeline documentation](README.md) apply. + +### Variables +There are five primary environment variables relevant to stability testing: +```bash +PINNED='true|false' # whether to perform the test with pinned dependencies, or default dependencies +ROUNDS='ℕ' # natural number defining the number of gated rounds of tests to generate +ROUND_SIZE='ℕ' # number of test steps to generate per operating system, per round +SKIP_MAC='true|false' # conserve finite macOS Anka agents by excluding them from your testing +TEST='name' # PCRE expression defining the tests to run, preceded by '^' and followed by '$' +``` +The `TEST` variable is parsed as [pearl-compatible regular expression](https://www.debuggex.com/cheatsheet/regex/pcre) where the expression in `TEST` is preceded by `^` and followed by `$`. To specify one test, set `TEST` equal to the test name (e.g. `TEST='read_only_query'`). Specify two tests as `TEST='(nodeos_short_fork_take_over_lr_test|read_only_query)'`. Or, perhaps, you want all of the `restart_scenarios` tests. Then, you could define `TEST='restart-scenario-test-.*'` and Buildkite will generate `ROUND_SIZE` steps each round for each operating system for all three restart scenarios tests. + +### Runs +The number of total test runs will be: +```bash +RUNS = ROUNDS * ROUND_SIZE * OS_COUNT * TEST_COUNT # where: +OS_COUNT = 'ℕ' # the number of supported operating systems +TEST_COUNT = 'ℕ' # the number of tests matching the PCRE filter in TEST +``` + +### Examples +We recommend stability testing one test per build with two builds per test, on Linux at first. Kick off one pinned build on Linux... +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_MAC='true' +TEST='read_only_query' +``` +...and one unpinned build on Linux: +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_MAC='true' +TEST='read_only_query' +``` +Once the Linux runs have proven stable, and if instability was observed on macOS, kick off two equivalent builds on macOS instead of Linux. One pinned build on macOS... +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_LINUX='true' +SKIP_MAC='false' +TEST='read_only_query' +``` +...and one unpinned build on macOS: +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_LINUX='true' +SKIP_MAC='false' +TEST='read_only_query' +``` +If these runs are against `eos:develop` and `develop` has five supported operating systems, this pattern would consist of 2,100 runs per test across all four builds. If the runs are against `eos:release/2.1.x` which, at the time of this writing, supports eight operating systems, this pattern would consist of 3,360 runs per test across all four builds. This gives you and your team strong confidence that any test instability occurs less than 1% of the time. + +# See Also +- Buildkite + - [DevDocs](https://github.com/EOSIO/devdocs/wiki/Buildkite) + - [EOSIO Pipelines](https://github.com/EOSIO/eos/blob/HEAD/.cicd/README.md) + - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) +- [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel + +
diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 43fd1b9cd6b..142cba2e964 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -15,6 +15,7 @@ export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pi if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' + [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]] && cat .cicd/eosio-test-stability.md | buildkite-agent annotate --append --style 'info' --context 'test-stability' fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" # guard against accidentally spawning too many jobs From 42d9c65e5c2e31a96313f1dac18ccc92e2e53a75 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 21:47:09 -0400 Subject: [PATCH 15/19] Include test metrics step in test stability runs --- .cicd/generate-pipeline.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 142cba2e964..98635a794f2 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -530,7 +530,7 @@ EOF fi fi # pipeline tail -[[ -z "$TEST" ]] && cat < Date: Fri, 25 Jun 2021 21:52:56 -0400 Subject: [PATCH 16/19] Update documentation links --- .cicd/generate-pipeline.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 98635a794f2..67539158ebe 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -10,11 +10,12 @@ export PLATFORMS_JSON_ARRAY='[]' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation -export DOCS_URL="https://github.com/EOSIO/eos/blob/$(git rev-parse HEAD)/.cicd/README.md" +export DOCS_URL="https://github.com/EOSIO/eos/blob/$(git rev-parse HEAD)/.cicd" export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then - echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' + echo "This documentation is also available on [GitHub]($DOCS_URL/README.md)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' + echo "This documentation is also available on [GitHub]($DOCS_URL/eosio-test-stability.md)." | buildkite-agent annotate --append --style 'info' --context 'test-stability' [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]] && cat .cicd/eosio-test-stability.md | buildkite-agent annotate --append --style 'info' --context 'test-stability' fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" From d9d230379f8d6fce180365efe88334cd39a1124c Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 21:56:56 -0400 Subject: [PATCH 17/19] Fix erroneous annotation --- .cicd/generate-pipeline.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 67539158ebe..c4d9e2d92b6 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -15,8 +15,10 @@ export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pi if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL/README.md)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' - echo "This documentation is also available on [GitHub]($DOCS_URL/eosio-test-stability.md)." | buildkite-agent annotate --append --style 'info' --context 'test-stability' - [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]] && cat .cicd/eosio-test-stability.md | buildkite-agent annotate --append --style 'info' --context 'test-stability' + if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then + echo "This documentation is also available on [GitHub]($DOCS_URL/eosio-test-stability.md)." | buildkite-agent annotate --append --style 'info' --context 'test-stability' + cat .cicd/eosio-test-stability.md | buildkite-agent annotate --append --style 'info' --context 'test-stability' + fi fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" # guard against accidentally spawning too many jobs From cc81e21ed659d30fe43fc0299ef9a77a2708b1fe Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 28 Jun 2021 12:18:47 -0400 Subject: [PATCH 18/19] Allow $TEST to be overidden by $1 in test scripts --- scripts/long-running-test.sh | 2 +- scripts/serial-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/long-running-test.sh b/scripts/long-running-test.sh index a3ab7bad43b..1edd082a932 100755 --- a/scripts/long-running-test.sh +++ b/scripts/long-running-test.sh @@ -3,7 +3,7 @@ set -eo pipefail # variables echo "--- $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" GIT_ROOT="$(dirname $BASH_SOURCE[0])/.." -[[ -z "$TEST" ]] && export TEST="$1" +[[ -z "$1" ]] || export TEST="$1" if [[ "$(uname)" == 'Linux' ]]; then . /etc/os-release if [[ "$ID" == 'centos' ]]; then diff --git a/scripts/serial-test.sh b/scripts/serial-test.sh index 8c46d2a7506..8a7f047840e 100755 --- a/scripts/serial-test.sh +++ b/scripts/serial-test.sh @@ -3,7 +3,7 @@ set -eo pipefail # variables echo "--- $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" GIT_ROOT="$(dirname $BASH_SOURCE[0])/.." -[[ -z "$TEST" ]] && export TEST="$1" +[[ -z "$1" ]] || export TEST="$1" if [[ "$(uname)" == 'Linux' ]]; then . /etc/os-release if [[ "$ID" == 'centos' ]]; then From 98fb612dae9f22e531a3fa0b96b1a539a9e8692a Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 28 Jun 2021 12:28:24 -0400 Subject: [PATCH 19/19] Add TIMEOUT to eosio-test-stability documentation --- .cicd/eosio-test-stability.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/eosio-test-stability.md b/.cicd/eosio-test-stability.md index 4000c87a406..ec9462eff5b 100644 --- a/.cicd/eosio-test-stability.md +++ b/.cicd/eosio-test-stability.md @@ -22,6 +22,7 @@ ROUNDS='ℕ' # natural number defining the number of gated rounds of ROUND_SIZE='ℕ' # number of test steps to generate per operating system, per round SKIP_MAC='true|false' # conserve finite macOS Anka agents by excluding them from your testing TEST='name' # PCRE expression defining the tests to run, preceded by '^' and followed by '$' +TIMEOUT='ℕ' # set timeout in minutes for all Buildkite steps ``` The `TEST` variable is parsed as [pearl-compatible regular expression](https://www.debuggex.com/cheatsheet/regex/pcre) where the expression in `TEST` is preceded by `^` and followed by `$`. To specify one test, set `TEST` equal to the test name (e.g. `TEST='read_only_query'`). Specify two tests as `TEST='(nodeos_short_fork_take_over_lr_test|read_only_query)'`. Or, perhaps, you want all of the `restart_scenarios` tests. Then, you could define `TEST='restart-scenario-test-.*'` and Buildkite will generate `ROUND_SIZE` steps each round for each operating system for all three restart scenarios tests.