Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Agent priority queues #7898

Closed
wants to merge 3 commits into from
Closed

Agent priority queues #7898

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2f043bf
Upload generated pipeline as a buildkite artifact
knmorgan Sep 12, 2019
82af2bf
Add concurrency groups for scheduled builds
knmorgan Sep 12, 2019
bdac095
Resolve merge conflicts with develop
kj4ezj Sep 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 58 additions & 13 deletions .cicd/generate-pipeline.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::
export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40G'}
export PLATFORMS_JSON_ARRAY='[]'
[[ -z "$ROUNDS" ]] && export ROUNDS='1'
# read .cicd/platforms
LINUX_CONCURRENCY='8'
MAC_CONCURRENCY='2'
LINUX_CONCURRENCY_GROUP='eos-scheduled-build'
MAC_CONCURRENCY_GROUP='eos-scheduled-build-mac'
for FILE in $(ls $CICD_DIR/platforms); do
# skip mac or linux by not even creating the json block
( [[ $SKIP_MAC == true ]] && [[ $FILE =~ 'macos' ]] ) && continue
Expand Down Expand Up @@ -79,12 +82,15 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
EOF
fi
done
BUILDKITE_AGENT_QUEUE='automation-eks-eos-builder-fleet'
echo ' - wait'
echo ''
# build steps
echo ' # builds'
echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
CONCURRENCY=$LINUX_CONCURRENCY
CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
cat <<EOF
- label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build"
command:
Expand All @@ -94,12 +100,14 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
BUILDKITE_AGENT_ACCESS_TOKEN:
agents:
queue: "automation-eos-builder-fleet"
queue: "$BUILDKITE_AGENT_QUEUE"
timeout: ${TIMEOUT:-180}
skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_BUILD}

EOF
else
CONCURRENCY=$MAC_CONCURRENCY
CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
cat <<EOF
- label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build"
command:
Expand All @@ -120,10 +128,16 @@ EOF
timeout: ${TIMEOUT:-180}
agents: "queue=mac-anka-large-node-fleet"
skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_BUILD}

EOF
fi
if [ "$BUILDKITE_SOURCE" = 'schedule' ]; then
cat <<EOF
concurrency: ${CONCURRENCY}
concurrency_group: ${CONCURRENCY_GROUP}
EOF
fi
done
echo
echo ' - wait'
echo ''
# tests
Expand All @@ -135,6 +149,8 @@ for ROUND in $(seq 1 $ROUNDS); do
echo ' # parallel tests'
echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do
if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
CONCURRENCY=$LINUX_CONCURRENCY
CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
cat <<EOF
- label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Unit Tests"
command:
Expand All @@ -144,12 +160,14 @@ for ROUND in $(seq 1 $ROUNDS); do
IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
BUILDKITE_AGENT_ACCESS_TOKEN:
agents:
queue: "automation-eos-builder-fleet"
queue: "$BUILDKITE_AGENT_QUEUE"
timeout: ${TIMEOUT:-10}
skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS}

EOF
else
CONCURRENCY=$MAC_CONCURRENCY
CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
cat <<EOF
- label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Unit Tests"
command:
Expand All @@ -168,9 +186,15 @@ EOF
timeout: ${TIMEOUT:-20}
agents: "queue=mac-anka-node-fleet"
skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS}

EOF
fi
if [ "$BUILDKITE_SOURCE" = "schedule" ]; then
cat <<EOF
concurrency: ${CONCURRENCY}
concurrency_group: ${CONCURRENCY_GROUP}
EOF
fi
echo
done
# serial tests
echo ' # serial tests'
Expand All @@ -179,6 +203,8 @@ EOF
SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }')"
for TEST_NAME in $SERIAL_TESTS; do
if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
CONCURRENCY=$LINUX_CONCURRENCY
CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
cat <<EOF
- label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
command:
Expand All @@ -188,12 +214,14 @@ EOF
IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
BUILDKITE_AGENT_ACCESS_TOKEN:
agents:
queue: "automation-eos-builder-fleet"
queue: "$BUILDKITE_AGENT_QUEUE"
timeout: ${TIMEOUT:-20}
skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_SERIAL_TESTS}

EOF
else
CONCURRENCY=$MAC_CONCURRENCY
CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
cat <<EOF
- label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
command:
Expand All @@ -215,6 +243,13 @@ EOF

EOF
fi
if [ "$BUILDKITE_SOURCE" = "schedule" ]; then
cat <<EOF
concurrency: ${CONCURRENCY}
concurrency_group: ${CONCURRENCY_GROUP}
EOF
fi
echo
done
IFS=$nIFS
done
Expand All @@ -225,6 +260,8 @@ EOF
LR_TESTS="$(cat tests/CMakeLists.txt | grep long_running_tests | grep -v "^#" | awk -F" " '{ print $2 }')"
for TEST_NAME in $LR_TESTS; do
if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
CONCURRENCY=$LINUX_CONCURRENCY
CONCURRENCY_GROUP=$LINUX_CONCURRENCY_GROUP
cat <<EOF
- label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
command:
Expand All @@ -234,12 +271,14 @@ EOF
IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
BUILDKITE_AGENT_ACCESS_TOKEN:
agents:
queue: "automation-eos-builder-fleet"
queue: "$BUILDKITE_AGENT_QUEUE"
timeout: ${TIMEOUT:-180}
skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_LONG_RUNNING_TESTS:-true}

EOF
else
CONCURRENCY=$MAC_CONCURRENCY
CONCURRENCY_GROUP=$MAC_CONCURRENCY_GROUP
cat <<EOF
- label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
command:
Expand All @@ -258,9 +297,15 @@ EOF
timeout: ${TIMEOUT:-180}
agents: "queue=mac-anka-node-fleet"
skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_LONG_RUNNING_TESTS:-true}

EOF
fi
if [ "$BUILDKITE_SOURCE" = "schedule" ]; then
cat <<EOF
concurrency: ${CONCURRENCY}
concurrency_group: ${CONCURRENCY_GROUP}
EOF
fi
echo
done
IFS=$nIFS
done
Expand All @@ -282,7 +327,7 @@ cat <<EOF
echo '+++ :javascript: Running test-metrics.js'
node --max-old-space-size=32768 test-metrics.js
agents:
queue: "automation-eos-builder-fleet"
queue: "$BUILDKITE_AGENT_QUEUE"
timeout: ${TIMEOUT:-10}
soft_fail: true

Expand All @@ -299,7 +344,7 @@ cat <<EOF
OS: "el7" # OS and PKGTYPE required for lambdas
PKGTYPE: "rpm"
agents:
queue: "automation-eos-builder-fleet"
queue: "$BUILDKITE_AGENT_QUEUE"
timeout: ${TIMEOUT:-10}
skip: ${SKIP_CENTOS_7_6}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX}

Expand All @@ -313,7 +358,7 @@ cat <<EOF
OS: "ubuntu-16.04" # OS and PKGTYPE required for lambdas
PKGTYPE: "deb"
agents:
queue: "automation-eos-builder-fleet"
queue: "$BUILDKITE_AGENT_QUEUE"
timeout: ${TIMEOUT:-10}
skip: ${SKIP_UBUNTU_16_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX}

Expand All @@ -327,7 +372,7 @@ cat <<EOF
OS: "ubuntu-18.04" # OS and PKGTYPE required for lambdas
PKGTYPE: "deb"
agents:
queue: "automation-eos-builder-fleet"
queue: "$BUILDKITE_AGENT_QUEUE"
timeout: ${TIMEOUT:-10}
skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX}

Expand Down Expand Up @@ -356,7 +401,7 @@ cat <<EOF
IMAGE_TAG: "ubuntu-18.04-unpinned"
BUILDKITE_AGENT_ACCESS_TOKEN:
agents:
queue: "automation-eos-builder-fleet"
queue: "$BUILDKITE_AGENT_QUEUE"
timeout: ${TIMEOUT:-30}
skip: ${SKIP_CONTRACT_BUILDER}${SKIP_LINUX}

Expand Down
9 changes: 6 additions & 3 deletions .cicd/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
steps:
- wait

- label: ":pipeline: Generate Pipeline Steps"
command: "./.cicd/generate-pipeline.sh | buildkite-agent pipeline upload"
- label: ":pipeline: Generate Pipeline Steps"
command:
- "./.cicd/generate-pipeline.sh > generated-pipeline.yml"
- "buildkite-agent pipeline upload < generated-pipeline.yml"
- "buildkite-agent artifact upload generated-pipeline.yml"
agents:
queue: "automation-basic-builder-fleet"
timeout: ${TIMEOUT:-10}
timeout: ${TIMEOUT:-10}