From e945412ea879df14fa7195754df3f8b8ba964bc4 Mon Sep 17 00:00:00 2001
From: Justin Brown <justin.brown@block.one>
Date: Thu, 12 Sep 2019 18:57:29 -0400
Subject: [PATCH 1/8] adds support for builder priority queues

---
 .cicd/generate-pipeline.sh | 75 +++++++++++++++++++++++++++++---------
 .cicd/pipeline.yml         |  7 +++-
 2 files changed, 63 insertions(+), 19 deletions(-)

diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh
index 1e11060236e..08d90ff6920 100755
--- a/.cicd/generate-pipeline.sh
+++ b/.cicd/generate-pipeline.sh
@@ -6,7 +6,10 @@ export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::
 export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40G'}
 export PLATFORMS_JSON_ARRAY='[]'
 [[ -z "$ROUNDS" ]] && export ROUNDS='1'
-# read .cicd/platforms
+LINUX_CONCURRENCY='8'
+MAC_CONCURRENCY='2'
+LINUX_CONCURRENCY_GROUP='eos-scheduled-build'
+MAC_CONCURRENCY_GROUP='eos-scheduled-build-mac'
 for FILE in $(ls $CICD_DIR/platforms); do
     # skip mac or linux by not even creating the json block
     ( [[ $SKIP_MAC == true ]] && [[ $FILE =~ 'macos' ]] ) && continue
@@ -79,12 +82,15 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
 EOF
     fi
 done
+BUILDKITE_AGENT_QUEUE='automation-eks-eos-builder-fleet'
 echo '  - wait'
 echo ''
 # build steps
 echo '    # builds'
 echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
     if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
+        CONCURRENCY=$LINUX_CONCURRENCY
+        CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
         cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build"
     command:
@@ -94,12 +100,13 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
       IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
       BUILDKITE_AGENT_ACCESS_TOKEN:
     agents:
-      queue: "automation-eos-builder-fleet"
+      queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-180}
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_BUILD}
-
 EOF
     else
+        CONCURRENCY=$MAC_CONCURRENCY
+        CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
         cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build"
     command:
@@ -120,9 +127,15 @@ EOF
     timeout: ${TIMEOUT:-180}
     agents: "queue=mac-anka-large-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_BUILD}
-
 EOF
     fi
+    if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
+        cat <<EOF
+    concurrency: ${CONCURRENCY}
+    concurrency_group: ${CONCURRENCY_GROUP}
+EOF
+    fi
+    echo
 done
 echo '  - wait'
 echo ''
@@ -135,6 +148,8 @@ for ROUND in $(seq 1 $ROUNDS); do
     echo '    # parallel tests'
     echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
         if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
+            CONCURRENCY=$LINUX_CONCURRENCY
+            CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
             cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Unit Tests"
     command:
@@ -144,12 +159,13 @@ for ROUND in $(seq 1 $ROUNDS); do
       IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
       BUILDKITE_AGENT_ACCESS_TOKEN:
     agents:
-      queue: "automation-eos-builder-fleet"
+      queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-10}
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS}
-
 EOF
         else
+            CONCURRENCY=$MAC_CONCURRENCY
+            CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
             cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Unit Tests"
     command:
@@ -168,9 +184,15 @@ EOF
     timeout: ${TIMEOUT:-20}
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS}
-
 EOF
         fi
+        if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
+            cat <<EOF
+    concurrency: ${CONCURRENCY}
+    concurrency_group: ${CONCURRENCY_GROUP}
+EOF
+        fi
+        echo
     done
     # serial tests
     echo '    # serial tests'
@@ -179,6 +201,8 @@ EOF
         SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }')"
         for TEST_NAME in $SERIAL_TESTS; do
             if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
+                CONCURRENCY=$LINUX_CONCURRENCY
+                CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
                 cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
@@ -188,12 +212,13 @@ EOF
       IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
       BUILDKITE_AGENT_ACCESS_TOKEN:
     agents:
-      queue: "automation-eos-builder-fleet"
+      queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-20}
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_SERIAL_TESTS}
-
 EOF
             else
+                CONCURRENCY=$MAC_CONCURRENCY
+                CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
                 cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
@@ -215,6 +240,13 @@ EOF
 
 EOF
             fi
+            if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
+                cat <<EOF
+    concurrency: ${CONCURRENCY}
+    concurrency_group: ${CONCURRENCY_GROUP}
+EOF
+            fi
+            echo
         done
         IFS=$nIFS
     done
@@ -225,6 +257,8 @@ EOF
         LR_TESTS="$(cat tests/CMakeLists.txt | grep long_running_tests | grep -v "^#" | awk -F" " '{ print $2 }')"
         for TEST_NAME in $LR_TESTS; do
             if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
+                CONCURRENCY=$LINUX_CONCURRENCY
+                CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
                 cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
@@ -234,12 +268,13 @@ EOF
       IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
       BUILDKITE_AGENT_ACCESS_TOKEN:
     agents:
-      queue: "automation-eos-builder-fleet"
+      queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-180}
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_LONG_RUNNING_TESTS:-true}
-
 EOF
             else
+                CONCURRENCY=$MAC_CONCURRENCY
+                CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
                 cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
@@ -258,9 +293,15 @@ EOF
     timeout: ${TIMEOUT:-180}
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_LONG_RUNNING_TESTS:-true}
-
 EOF
             fi
+            if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
+                cat <<EOF
+    concurrency: ${CONCURRENCY}
+    concurrency_group: ${CONCURRENCY_GROUP}
+EOF
+            fi
+            echo
         done
         IFS=$nIFS
     done
@@ -282,7 +323,7 @@ cat <<EOF
       echo '+++ :javascript: Running test-metrics.js'
       node --max-old-space-size=32768 test-metrics.js
     agents:
-      queue: "automation-eos-builder-fleet"
+      queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-10}
     soft_fail: true
 
@@ -299,7 +340,7 @@ cat <<EOF
       OS: "el7" # OS and PKGTYPE required for lambdas
       PKGTYPE: "rpm"
     agents:
-      queue: "automation-eos-builder-fleet"
+      queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-10}
     skip: ${SKIP_CENTOS_7_6}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX}
 
@@ -313,7 +354,7 @@ cat <<EOF
       OS: "ubuntu-16.04" # OS and PKGTYPE required for lambdas
       PKGTYPE: "deb"
     agents:
-      queue: "automation-eos-builder-fleet"
+      queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-10}
     skip: ${SKIP_UBUNTU_16_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX}
 
@@ -327,7 +368,7 @@ cat <<EOF
       OS: "ubuntu-18.04" # OS and PKGTYPE required for lambdas
       PKGTYPE: "deb"
     agents:
-      queue: "automation-eos-builder-fleet"
+      queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-10}
     skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX}
 
@@ -356,7 +397,7 @@ cat <<EOF
       IMAGE_TAG: "ubuntu-18.04-unpinned"
       BUILDKITE_AGENT_ACCESS_TOKEN:
     agents:
-      queue: "automation-eos-builder-fleet"
+      queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-30}
     skip: ${SKIP_CONTRACT_BUILDER}${SKIP_LINUX}
 
diff --git a/.cicd/pipeline.yml b/.cicd/pipeline.yml
index 50cf1a713d7..900aea633e0 100644
--- a/.cicd/pipeline.yml
+++ b/.cicd/pipeline.yml
@@ -1,8 +1,11 @@
 steps:
   - wait
 
-  - label: ":pipeline: Generate Pipeline Steps"  
-    command: "./.cicd/generate-pipeline.sh | buildkite-agent pipeline upload"
+  - label: ":pipeline: Generate Pipeline Steps"
+    command:
+      - "./.cicd/generate-pipeline.sh > generated-pipeline.yml"
+      - "buildkite-agent pipeline upload < generated-pipeline.yml"
+      - "buildkite-agent artifact upload generated-pipeline.yml"
     agents:
       queue: "automation-basic-builder-fleet"
     timeout: ${TIMEOUT:-10}
\ No newline at end of file

From bf83cf0fda1a32ebb80c5eaaf87cccce6ecf94e9 Mon Sep 17 00:00:00 2001
From: Justin Brown <justin.brown@block.one>
Date: Thu, 12 Sep 2019 20:59:39 -0400
Subject: [PATCH 2/8] adds fix for unpinned builds on 18.04

---
 .cicd/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cicd/build.sh b/.cicd/build.sh
index 2688b65413a..cbe2eb50785 100755
--- a/.cicd/build.sh
+++ b/.cicd/build.sh
@@ -32,7 +32,7 @@ else # Linux
         PRE_COMMANDS="$PRE_COMMANDS && source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable && export PATH=/usr/lib64/ccache:\\\$PATH"
     elif [[ $IMAGE_TAG == 'ubuntu-18.04-unpinned' ]]; then
         PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH"
-        CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'"
+        CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang' -DLLVM_DIR='/usr/lib/llvm-7'"
     fi
     BUILD_COMMANDS="cmake $CMAKE_EXTRAS .. && make -j$JOBS"
     # Docker Commands

From c5fdc66e7492884941b701b072d6bacfd4a62591 Mon Sep 17 00:00:00 2001
From: Justin Brown <justin.brown@block.one>
Date: Thu, 12 Sep 2019 21:27:57 -0400
Subject: [PATCH 3/8] corrects path to llvm

---
 .cicd/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cicd/build.sh b/.cicd/build.sh
index cbe2eb50785..22a8e825006 100755
--- a/.cicd/build.sh
+++ b/.cicd/build.sh
@@ -32,7 +32,7 @@ else # Linux
         PRE_COMMANDS="$PRE_COMMANDS && source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable && export PATH=/usr/lib64/ccache:\\\$PATH"
     elif [[ $IMAGE_TAG == 'ubuntu-18.04-unpinned' ]]; then
         PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH"
-        CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang' -DLLVM_DIR='/usr/lib/llvm-7'"
+        CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang' -DLLVM_DIR='/usr/lib/llvm-7/lib/cmake/llvm'"
     fi
     BUILD_COMMANDS="cmake $CMAKE_EXTRAS .. && make -j$JOBS"
     # Docker Commands

From 7fbf2657299ba7434108559f08bf4f60af3fd1c1 Mon Sep 17 00:00:00 2001
From: Zach Butler <zach.butler@block.one>
Date: Fri, 13 Sep 2019 12:07:52 -0400
Subject: [PATCH 4/8] Remove FOR loop added for eosio-test-stability pipeline

---
 .cicd/generate-pipeline.sh | 133 +++++++++++++++++--------------------
 1 file changed, 62 insertions(+), 71 deletions(-)

diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh
index 08d90ff6920..249179773a1 100755
--- a/.cicd/generate-pipeline.sh
+++ b/.cicd/generate-pipeline.sh
@@ -5,7 +5,6 @@ set -eo pipefail
 export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::brew::buildkite-agent'}
 export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40G'}
 export PLATFORMS_JSON_ARRAY='[]'
-[[ -z "$ROUNDS" ]] && export ROUNDS='1'
 LINUX_CONCURRENCY='8'
 MAC_CONCURRENCY='2'
 LINUX_CONCURRENCY_GROUP='eos-scheduled-build'
@@ -140,17 +139,13 @@ done
 echo '  - wait'
 echo ''
 # tests
-IFS=$oIFS
-for ROUND in $(seq 1 $ROUNDS); do
-    IFS=$''
-    echo "    # round $ROUND of $ROUNDS"
-    # parallel tests
-    echo '    # parallel tests'
-    echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
-        if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
-            CONCURRENCY=$LINUX_CONCURRENCY
-            CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
-            cat <<EOF
+# parallel tests
+echo '    # parallel tests'
+echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
+    if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
+        CONCURRENCY=$LINUX_CONCURRENCY
+        CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
+        cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Unit Tests"
     command:
       - "buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz"
@@ -162,11 +157,12 @@ for ROUND in $(seq 1 $ROUNDS); do
       queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-10}
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS}
+
 EOF
-        else
-            CONCURRENCY=$MAC_CONCURRENCY
-            CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
-            cat <<EOF
+    else
+        CONCURRENCY=$MAC_CONCURRENCY
+        CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
+        cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Unit Tests"
     command:
       - "git clone \$BUILDKITE_REPO eos && cd eos && git checkout \$BUILDKITE_COMMIT && git submodule update --init --recursive"
@@ -185,25 +181,25 @@ EOF
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS}
 EOF
-        fi
-        if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
-            cat <<EOF
+    fi
+    if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
+        cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}
 EOF
-        fi
-        echo
-    done
-    # serial tests
-    echo '    # serial tests'
-    echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
-        IFS=$oIFS
-        SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }')"
-        for TEST_NAME in $SERIAL_TESTS; do
-            if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
-                CONCURRENCY=$LINUX_CONCURRENCY
-                CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
-                cat <<EOF
+    fi
+    echo
+done
+# serial tests
+echo '    # serial tests'
+echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
+    IFS=$oIFS
+    SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }')"
+    for TEST_NAME in $SERIAL_TESTS; do
+        if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
+            CONCURRENCY=$LINUX_CONCURRENCY
+            CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
+            cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
       - "buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz"
@@ -216,10 +212,10 @@ EOF
     timeout: ${TIMEOUT:-20}
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_SERIAL_TESTS}
 EOF
-            else
-                CONCURRENCY=$MAC_CONCURRENCY
-                CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
-                cat <<EOF
+        else
+            CONCURRENCY=$MAC_CONCURRENCY
+            CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
+            cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
       - "git clone \$BUILDKITE_REPO eos && cd eos && git checkout \$BUILDKITE_COMMIT && git submodule update --init --recursive"
@@ -239,27 +235,27 @@ EOF
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_SERIAL_TESTS}
 
 EOF
-            fi
-            if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
-                cat <<EOF
+        fi
+        if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
+            cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}
 EOF
-            fi
-            echo
-        done
-        IFS=$nIFS
+        fi
+        echo
     done
-    # long-running tests
-    echo '    # long-running tests'
-    echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
-        IFS=$oIFS
-        LR_TESTS="$(cat tests/CMakeLists.txt | grep long_running_tests | grep -v "^#" | awk -F" " '{ print $2 }')"
-        for TEST_NAME in $LR_TESTS; do
-            if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
-                CONCURRENCY=$LINUX_CONCURRENCY
-                CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
-                cat <<EOF
+    IFS=$nIFS
+done
+# long-running tests
+echo '    # long-running tests'
+echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
+    IFS=$oIFS
+    LR_TESTS="$(cat tests/CMakeLists.txt | grep long_running_tests | grep -v "^#" | awk -F" " '{ print $2 }')"
+    for TEST_NAME in $LR_TESTS; do
+        if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
+            CONCURRENCY=$LINUX_CONCURRENCY
+            CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
+            cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
       - "buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz"
@@ -271,11 +267,12 @@ EOF
       queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-180}
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_LONG_RUNNING_TESTS:-true}
+
 EOF
-            else
-                CONCURRENCY=$MAC_CONCURRENCY
-                CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
-                cat <<EOF
+        else
+            CONCURRENCY=$MAC_CONCURRENCY
+            CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
+            cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
       - "git clone \$BUILDKITE_REPO eos && cd eos && git checkout \$BUILDKITE_COMMIT && git submodule update --init --recursive"
@@ -294,23 +291,18 @@ EOF
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_LONG_RUNNING_TESTS:-true}
 EOF
-            fi
-            if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
-                cat <<EOF
+        fi
+        if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
+            cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}
 EOF
-            fi
-            echo
-        done
-        IFS=$nIFS
+        fi
+        echo
     done
-    IFS=$oIFS
-    if [[ "$ROUND" != "$ROUNDS" ]]; then
-        echo '  - wait'
-        echo ''
-    fi
+    IFS=$nIFS
 done
+IFS=$oIFS
 # pipeline tail
 cat <<EOF
   - wait:
@@ -418,5 +410,4 @@ cat <<EOF
     timeout: "${TIMEOUT:-5}"
     skip: ${SKIP_PACKAGE_BUILDER}${SKIP_MAC}${SKIP_MACOS_10_14}
 
-EOF
-IFS=$oIFS
\ No newline at end of file
+EOF
\ No newline at end of file

From 47559d396c1082632a8759a4f3e13646c002e42a Mon Sep 17 00:00:00 2001
From: Zach Butler <zach.butler@block.one>
Date: Fri, 13 Sep 2019 13:00:21 -0400
Subject: [PATCH 5/8] Add concurrency groups to ALL eosio-test-stability builds

---
 .cicd/generate-pipeline.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh
index 249179773a1..1d31375eb32 100755
--- a/.cicd/generate-pipeline.sh
+++ b/.cicd/generate-pipeline.sh
@@ -182,7 +182,7 @@ EOF
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS}
 EOF
     fi
-    if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
+    if [[ "$BUILDKITE_SOURCE" = 'schedule' || "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then
         cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}
@@ -236,7 +236,7 @@ EOF
 
 EOF
         fi
-        if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
+        if [[ "$BUILDKITE_SOURCE" = 'schedule' || "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then
             cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}
@@ -292,7 +292,7 @@ EOF
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_LONG_RUNNING_TESTS:-true}
 EOF
         fi
-        if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
+        if [[ "$BUILDKITE_SOURCE" = 'schedule' || "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then
             cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}

From 43c1297fc87dbab2047df10f8012ae591432a8f5 Mon Sep 17 00:00:00 2001
From: Zach Butler <zach.butler@block.one>
Date: Fri, 13 Sep 2019 13:02:43 -0400
Subject: [PATCH 6/8] Add "parallelism" for eosio-test-stability pipeline

---
 .cicd/generate-pipeline.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh
index 1d31375eb32..9a9fead9daa 100755
--- a/.cicd/generate-pipeline.sh
+++ b/.cicd/generate-pipeline.sh
@@ -157,6 +157,7 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
       queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-10}
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS}
+    parallelism: \${ROUNDS:-1}
 
 EOF
     else
@@ -180,6 +181,7 @@ EOF
     timeout: ${TIMEOUT:-20}
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS}
+    parallelism: \${ROUNDS:-1}
 EOF
     fi
     if [[ "$BUILDKITE_SOURCE" = 'schedule' || "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then
@@ -211,6 +213,7 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
       queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-20}
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_SERIAL_TESTS}
+    parallelism: \${ROUNDS:-1}
 EOF
         else
             CONCURRENCY=$MAC_CONCURRENCY
@@ -233,7 +236,7 @@ EOF
     timeout: ${TIMEOUT:-20}
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_SERIAL_TESTS}
-
+    parallelism: \${ROUNDS:-1}
 EOF
         fi
         if [[ "$BUILDKITE_SOURCE" = 'schedule' || "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then
@@ -267,7 +270,7 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
       queue: "$BUILDKITE_AGENT_QUEUE"
     timeout: ${TIMEOUT:-180}
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_LONG_RUNNING_TESTS:-true}
-
+    parallelism: \${ROUNDS:-1}
 EOF
         else
             CONCURRENCY=$MAC_CONCURRENCY
@@ -290,6 +293,7 @@ EOF
     timeout: ${TIMEOUT:-180}
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_LONG_RUNNING_TESTS:-true}
+    parallelism: \${ROUNDS:-1}
 EOF
         fi
         if [[ "$BUILDKITE_SOURCE" = 'schedule' || "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then

From 2351a65eda32638d74fd63243da5cc9e0b4dfd01 Mon Sep 17 00:00:00 2001
From: Zach Butler <zach.butler@block.one>
Date: Fri, 13 Sep 2019 13:48:23 -0400
Subject: [PATCH 7/8] Increase concurrency for eosio-test-stability Pipeline

---
 .cicd/generate-pipeline.sh | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh
index 9a9fead9daa..fba50bbddec 100755
--- a/.cicd/generate-pipeline.sh
+++ b/.cicd/generate-pipeline.sh
@@ -5,10 +5,17 @@ set -eo pipefail
 export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::brew::buildkite-agent'}
 export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40G'}
 export PLATFORMS_JSON_ARRAY='[]'
-LINUX_CONCURRENCY='8'
-MAC_CONCURRENCY='2'
-LINUX_CONCURRENCY_GROUP='eos-scheduled-build'
-MAC_CONCURRENCY_GROUP='eos-scheduled-build-mac'
+if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then
+    LINUX_CONCURRENCY='50'
+    MAC_CONCURRENCY='2'
+    LINUX_CONCURRENCY_GROUP="$BUILDKITE_PIPELINE_SLUG-build-$BUILDKITE_BUILD_NUMBER"
+    MAC_CONCURRENCY_GROUP="$BUILDKITE_PIPELINE_SLUG-build-$BUILDKITE_BUILD_NUMBER-mac"
+else
+    LINUX_CONCURRENCY='8'
+    MAC_CONCURRENCY='2'
+    LINUX_CONCURRENCY_GROUP='eos-scheduled-build'
+    MAC_CONCURRENCY_GROUP='eos-scheduled-build-mac'
+fi
 for FILE in $(ls $CICD_DIR/platforms); do
     # skip mac or linux by not even creating the json block
     ( [[ $SKIP_MAC == true ]] && [[ $FILE =~ 'macos' ]] ) && continue

From 74249b1c316f907d0855cd60b04f0eed241deee3 Mon Sep 17 00:00:00 2001
From: Zach Butler <zach.butler@block.one>
Date: Fri, 13 Sep 2019 13:55:24 -0400
Subject: [PATCH 8/8] Fix missing comment

---
 .cicd/generate-pipeline.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh
index fba50bbddec..b2afaf6ebeb 100755
--- a/.cicd/generate-pipeline.sh
+++ b/.cicd/generate-pipeline.sh
@@ -16,6 +16,7 @@ else
     LINUX_CONCURRENCY_GROUP='eos-scheduled-build'
     MAC_CONCURRENCY_GROUP='eos-scheduled-build-mac'
 fi
+# read .cicd/platforms
 for FILE in $(ls $CICD_DIR/platforms); do
     # skip mac or linux by not even creating the json block
     ( [[ $SKIP_MAC == true ]] && [[ $FILE =~ 'macos' ]] ) && continue