From 9932668107c7c3e81a8dc30a66f891a30b2052f9 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Thu, 24 Jun 2021 13:24:30 -0400 Subject: [PATCH 01/19] Support running the pipeline upload script locally --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index ab2d53471a4..19f10f3b7d0 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -8,7 +8,7 @@ BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" -export RETRY="$(buildkite-agent meta-data get pipeline-upload-retries --default '0')" +export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' From 114af9f84eec51fd43784eec342b5e3ce753e54c Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Thu, 24 Jun 2021 13:30:30 -0400 Subject: [PATCH 02/19] Fix step retry count --- .cicd/generate-pipeline.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 19f10f3b7d0..e0f27677f32 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -13,6 +13,7 @@ if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' fi +[[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) From 63a4b71733c5d1f1f592879147da9bbf12b1c5f4 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 11:06:09 -0400 Subject: [PATCH 03/19] Fix alignment of "fi" statement --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index e0f27677f32..97169712e4a 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -460,7 +460,7 @@ EOF timeout: ${TIMEOUT:-180} EOF - fi + fi if [[ "$ROUND" != "$ROUNDS" ]]; then echo ' - wait' echo '' From 96b09ab352ae3a4422ed8d7042a341403cde6b5b Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 16:33:12 -0400 Subject: [PATCH 04/19] Alphabetize test steps --- .cicd/generate-pipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 97169712e4a..25f07a929b6 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -318,7 +318,7 @@ EOF echo ' # serial tests' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do IFS=$oIFS - SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }')" + SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }' | sort | uniq)" for TEST_NAME in $SERIAL_TESTS; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat < Date: Fri, 25 Jun 2021 17:25:13 -0400 Subject: [PATCH 05/19] Fix null SKIP variable --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 25f07a929b6..8f1fbcf0286 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -335,7 +335,7 @@ EOF manual: permit_on_passed: true timeout: ${TIMEOUT:-20} - skip: $(echo "$PLATFORM_JSON" | jq -r '.PLATFORM_SKIP_VAR | env[.]')${SKIP_SERIAL_TESTS} + skip: $(echo "$PLATFORM_JSON" | jq -r '.PLATFORM_SKIP_VAR | env[.] // empty')${SKIP_SERIAL_TESTS} EOF else From 40ec1c15543ff3ce4d85894e7dd8a7be54e79fc4 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 17:42:18 -0400 Subject: [PATCH 06/19] Only create serial test steps when TEST is non-empty --- .cicd/generate-pipeline.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 8f1fbcf0286..dcc3515efaf 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -203,7 +203,7 @@ if [[ "$DCMAKE_BUILD_TYPE" != 'Debug' ]]; then echo " # round $ROUND of $ROUNDS" # parallel tests echo ' # parallel tests' - echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do + [[ -z "$TEST" ]] && echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat < Date: Fri, 25 Jun 2021 17:45:12 -0400 Subject: [PATCH 07/19] Support PCRE test filter --- .cicd/generate-pipeline.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index dcc3515efaf..32fe5cb64c8 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -318,7 +318,11 @@ EOF echo ' # serial tests' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do IFS=$oIFS - SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }' | sort | uniq)" + if [[ -z "$TEST" ]]; then + SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq)" + else + SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq | grep -P "^$TEST$")" + fi for TEST_NAME in $SERIAL_TESTS; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat < Date: Fri, 25 Jun 2021 16:43:00 -0400 Subject: [PATCH 08/19] Support ROUND_SIZE --- .cicd/generate-pipeline.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 32fe5cb64c8..f24370562fb 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -4,6 +4,7 @@ set -eo pipefail . ./.cicd/helpers/general.sh export PLATFORMS_JSON_ARRAY='[]' [[ -z "$ROUNDS" ]] && export ROUNDS='1' +[[ -z "$ROUND_SIZE" ]] && export ROUND_SIZE='5' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation @@ -321,7 +322,7 @@ EOF if [[ -z "$TEST" ]]; then SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq)" else - SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq | grep -P "^$TEST$")" + SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq | grep -P "^$TEST$" | awk "{while(i++<$ROUND_SIZE)print;i=0}")" fi for TEST_NAME in $SERIAL_TESTS; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then From a53b1070d40ac76641e3420798ae85fa553e9fdc Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 17:52:49 -0400 Subject: [PATCH 09/19] Consistent TIMEOUT formatting --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index f24370562fb..52b2a3c15d3 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -679,7 +679,7 @@ fi - "buildkite-agent artifact upload eosio.rb" agents: queue: "automation-basic-builder-fleet" - timeout: "${TIMEOUT:-5}" + timeout: ${TIMEOUT:-5} skip: ${SKIP_PACKAGE_BUILDER}${SKIP_MAC}${SKIP_MACOS_10_15} - label: ":docker: :ubuntu: Docker - Build 18.04 Docker Image" From 08228400ec56ca686c59875f7116e42764c0c6d5 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 18:01:23 -0400 Subject: [PATCH 10/19] Put default ROUND_SIZE back to 1 because it is used for all builds now --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 52b2a3c15d3..b37bba2a72c 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -4,7 +4,7 @@ set -eo pipefail . ./.cicd/helpers/general.sh export PLATFORMS_JSON_ARRAY='[]' [[ -z "$ROUNDS" ]] && export ROUNDS='1' -[[ -z "$ROUND_SIZE" ]] && export ROUND_SIZE='5' +[[ -z "$ROUND_SIZE" ]] && export ROUND_SIZE='1' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation From 56af53afbe1495c625e581d47dd24a82b4c161ce Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 18:07:02 -0400 Subject: [PATCH 11/19] Remove extraneous newlines --- .cicd/generate-pipeline.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index b37bba2a72c..49d1b68c949 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -258,7 +258,6 @@ EOF EOF fi - echo done # wasm spec tests echo ' # wasm spec tests' @@ -313,7 +312,6 @@ EOF EOF fi - echo done # serial tests echo ' # serial tests' @@ -375,7 +373,6 @@ EOF EOF fi - echo done IFS=$nIFS done @@ -435,7 +432,6 @@ EOF EOF fi - echo done IFS=$nIFS done @@ -471,7 +467,6 @@ EOF echo '' fi done - # trigger eosio-lrt post pr if [[ -z "$TEST" && -z $BUILDKITE_TRIGGERED_FROM_BUILD_ID && $TRIGGER_JOB == "true" ]]; then if ( [[ ! $PINNED == false ]] ); then From 0309cb6994cdbbde75c6e158512dbffee709155d Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 18:07:45 -0400 Subject: [PATCH 12/19] Exclude docker build and install step when TEST is non-empty, too --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 49d1b68c949..a3d46d1956d 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -181,7 +181,7 @@ EOF EOF fi done -cat < Date: Fri, 25 Jun 2021 19:10:00 -0400 Subject: [PATCH 13/19] Add guard against users spawning too many jobs by accident --- .cicd/generate-pipeline.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index a3d46d1956d..994e3720d30 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -15,6 +15,19 @@ if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" +# guard against accidentally spawning too many jobs +if (( $ROUNDS > 1 || $ROUND_SIZE > 1 )) && [[ -z "$TEST" ]]; then + echo '+++ :no_entry: WARNING: Your parameters will spawn a very large number of jobs!' 1>&2 + echo "Setting ROUNDS='$ROUNDS' and/or ROUND_SIZE='$ROUND_SIZE' in the environment without also setting TEST to a specific test will cause ALL tests to be run $(( $ROUNDS * $ROUND_SIZE )) times, which will consume a large number of agents! We recommend doing stability testing on ONE test at a time. If you're certain you want to do this, set TEST='.*' to run all tests $(( $ROUNDS * $ROUND_SIZE )) times." 1>&2 + [[ "$BUILDKITE" == 'true' ]] && cat | buildkite-agent annotate --append --style 'error' --context 'no-TEST' <<-MD + Your build was cancelled because you set \`ROUNDS\` and/or \`ROUND_SIZE\` without also setting \`TEST\` in your build environment. This would cause each test to be run $(( $ROUNDS * $ROUND_SIZE )) times, which will consume a lot of Buildkite agents. + + We recommend stability testing one test at a time by setting \`TEST\` equal to the test name. Alternatively, you can specify a set of test names using [Perl-Compatible Regular Expressions](https://www.debuggex.com/cheatsheet/regex/pcre), where \`TEST\` is parsed as \`^${TEST}$\`. + + If you _really_ meant to run every test $(( $ROUNDS * $ROUND_SIZE )) times, set \`TEST='.*'\`. +MD + exit 255 +fi # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) From 5ab6d918bb5e13a3d85139a906562fdbef9d6bbb Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 19:56:06 -0400 Subject: [PATCH 14/19] If they want to run every test, just spawn the jobs like normal --- .cicd/generate-pipeline.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 994e3720d30..88a0b7a6ab6 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -27,6 +27,8 @@ if (( $ROUNDS > 1 || $ROUND_SIZE > 1 )) && [[ -z "$TEST" ]]; then If you _really_ meant to run every test $(( $ROUNDS * $ROUND_SIZE )) times, set \`TEST='.*'\`. MD exit 255 +elif [[ "$TEST" = '.*' ]]; then # if they want to run every test, just spawn the jobs like normal + unset TEST fi # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then From a45235e95cbb800c71762be6dbe5cdc0f7be0fc2 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 21:32:05 -0400 Subject: [PATCH 15/19] Fix link to current documentation --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 88a0b7a6ab6..156d2cde174 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -8,7 +8,7 @@ export PLATFORMS_JSON_ARRAY='[]' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation -export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" +export DOCS_URL="https://github.com/EOSIO/eos/blob/$(git rev-parse HEAD)/.cicd/README.md" export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' From ac544e0159ab8d11135ffc342e3e53f4ba88ff36 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 21:37:24 -0400 Subject: [PATCH 16/19] Update documentation --- .cicd/README.md | 2 + .cicd/eosio-test-stability.md | 80 +++++++++++++++++++++++++++++++++++ .cicd/generate-pipeline.sh | 1 + 3 files changed, 83 insertions(+) create mode 100644 .cicd/eosio-test-stability.md diff --git a/.cicd/README.md b/.cicd/README.md index 93ad073ae53..d2480f17427 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -94,12 +94,14 @@ Pipeline | Details [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | runs tests that need more time on merge commits [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | loads the current version of `nodeos` from state files generated by specific previous versions of `nodeos` in each [eosio](https://buildkite.com/EOSIO/eosio) build ([Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md)) [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | sync the current version of `nodeos` past genesis from peers on common public chains as a smoke test, for each [eosio](https://buildkite.com/EOSIO/eosio) build +[eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) | prove or disprove test stability by running a test thousands of times ## See Also - Buildkite - [DevDocs](https://github.com/EOSIO/devdocs/wiki/Buildkite) - [eosio-resume-from-state Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) + - [Stability Testing](https://github.com/EOSIO/eos/blob/HEAD/.cicd/eosio-test-stability.md) - [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel diff --git a/.cicd/eosio-test-stability.md b/.cicd/eosio-test-stability.md new file mode 100644 index 00000000000..4000c87a406 --- /dev/null +++ b/.cicd/eosio-test-stability.md @@ -0,0 +1,80 @@ +# Stability Testing +Stability testing of EOSIO unit and integration tests is done in the [eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) pipeline. It will take thousands of runs of any given test to identify it as "stable" or "unstable". Runs should be split evenly across "pinned" (fixed dependency version) and "unpinned" (default dependency version) builds because, sometimes, test instability is only expressed in one of these environments. Finally, stability testing should be performed on the Linux fleet first because this fleet is effectively infinite. Once stability is demonstrated on Linux, testing can be performed on the finite macOS Anka fleet. + +
+See More + +## Index +1. [Configuration](eosio-test-stability.md#configuration) + 1. [Variables](eosio-test-stability.md#variables) + 1. [Runs](eosio-test-stability.md#runs) + 1. [Examples](eosio-test-stability.md#examples) +1. [See Also](eosio-test-stability.md#see-also) + +## Configuration +The [eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) pipeline uses the same pipeline upload script as [eosio](https://buildkite.com/EOSIO/eosio), [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned), and [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt), so all variables from the [pipeline documentation](README.md) apply. + +### Variables +There are five primary environment variables relevant to stability testing: +```bash +PINNED='true|false' # whether to perform the test with pinned dependencies, or default dependencies +ROUNDS='ℕ' # natural number defining the number of gated rounds of tests to generate +ROUND_SIZE='ℕ' # number of test steps to generate per operating system, per round +SKIP_MAC='true|false' # conserve finite macOS Anka agents by excluding them from your testing +TEST='name' # PCRE expression defining the tests to run, preceded by '^' and followed by '$' +``` +The `TEST` variable is parsed as [pearl-compatible regular expression](https://www.debuggex.com/cheatsheet/regex/pcre) where the expression in `TEST` is preceded by `^` and followed by `$`. To specify one test, set `TEST` equal to the test name (e.g. `TEST='read_only_query'`). Specify two tests as `TEST='(nodeos_short_fork_take_over_lr_test|read_only_query)'`. Or, perhaps, you want all of the `restart_scenarios` tests. Then, you could define `TEST='restart-scenario-test-.*'` and Buildkite will generate `ROUND_SIZE` steps each round for each operating system for all three restart scenarios tests. + +### Runs +The number of total test runs will be: +```bash +RUNS = ROUNDS * ROUND_SIZE * OS_COUNT * TEST_COUNT # where: +OS_COUNT = 'ℕ' # the number of supported operating systems +TEST_COUNT = 'ℕ' # the number of tests matching the PCRE filter in TEST +``` + +### Examples +We recommend stability testing one test per build with two builds per test, on Linux at first. Kick off one pinned build on Linux... +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_MAC='true' +TEST='read_only_query' +``` +...and one unpinned build on Linux: +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_MAC='true' +TEST='read_only_query' +``` +Once the Linux runs have proven stable, and if instability was observed on macOS, kick off two equivalent builds on macOS instead of Linux. One pinned build on macOS... +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_LINUX='true' +SKIP_MAC='false' +TEST='read_only_query' +``` +...and one unpinned build on macOS: +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_LINUX='true' +SKIP_MAC='false' +TEST='read_only_query' +``` +If these runs are against `eos:develop` and `develop` has five supported operating systems, this pattern would consist of 2,100 runs per test across all four builds. If the runs are against `eos:release/2.1.x` which, at the time of this writing, supports eight operating systems, this pattern would consist of 3,360 runs per test across all four builds. This gives you and your team strong confidence that any test instability occurs less than 1% of the time. + +# See Also +- Buildkite + - [DevDocs](https://github.com/EOSIO/devdocs/wiki/Buildkite) + - [EOSIO Pipelines](https://github.com/EOSIO/eos/blob/HEAD/.cicd/README.md) + - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) +- [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel + +
diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 156d2cde174..aac61894723 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -13,6 +13,7 @@ export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pi if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' + [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]] && cat .cicd/eosio-test-stability.md | buildkite-agent annotate --append --style 'info' --context 'test-stability' fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" # guard against accidentally spawning too many jobs From 5647b12b148561c35e8a75c89f1f9060ed967b4d Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 21:47:09 -0400 Subject: [PATCH 17/19] Include test metrics step in test stability runs --- .cicd/generate-pipeline.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index aac61894723..80cb65eeefc 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -548,7 +548,7 @@ EOF fi fi # pipeline tail -[[ -z "$TEST" ]] && cat < Date: Fri, 25 Jun 2021 21:52:56 -0400 Subject: [PATCH 18/19] Update documentation links --- .cicd/generate-pipeline.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 80cb65eeefc..ca89ca0610c 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -8,11 +8,12 @@ export PLATFORMS_JSON_ARRAY='[]' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation -export DOCS_URL="https://github.com/EOSIO/eos/blob/$(git rev-parse HEAD)/.cicd/README.md" +export DOCS_URL="https://github.com/EOSIO/eos/blob/$(git rev-parse HEAD)/.cicd" export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then - echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' + echo "This documentation is also available on [GitHub]($DOCS_URL/README.md)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' + echo "This documentation is also available on [GitHub]($DOCS_URL/eosio-test-stability.md)." | buildkite-agent annotate --append --style 'info' --context 'test-stability' [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]] && cat .cicd/eosio-test-stability.md | buildkite-agent annotate --append --style 'info' --context 'test-stability' fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" From b98e8c4f06780339a2cf9bd7028ade0b25859b64 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 25 Jun 2021 21:56:56 -0400 Subject: [PATCH 19/19] Fix erroneous annotation --- .cicd/generate-pipeline.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index ca89ca0610c..b5295f7e6f0 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -13,8 +13,10 @@ export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pi if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL/README.md)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' - echo "This documentation is also available on [GitHub]($DOCS_URL/eosio-test-stability.md)." | buildkite-agent annotate --append --style 'info' --context 'test-stability' - [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]] && cat .cicd/eosio-test-stability.md | buildkite-agent annotate --append --style 'info' --context 'test-stability' + if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then + echo "This documentation is also available on [GitHub]($DOCS_URL/eosio-test-stability.md)." | buildkite-agent annotate --append --style 'info' --context 'test-stability' + cat .cicd/eosio-test-stability.md | buildkite-agent annotate --append --style 'info' --context 'test-stability' + fi fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" # guard against accidentally spawning too many jobs