Skip to content

Scheduled

Scheduled #14139

Workflow file for this run

name: Scheduled
on:
schedule:
- cron: '45 * * * *'
workflow_dispatch:
concurrency:
group: "scheduled"
cancel-in-progress: true
jobs:
check:
runs-on: ubuntu-latest
outputs:
configs: ${{ steps.repo_check.outputs.configs }}
steps:
- uses: actions/checkout@v4
- uses: mikefarah/yq@v4.35.1
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ vars.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- id: repo_check
env:
CONCURRENT_IMAGE_PULL: ${{ vars.CONCURRENT_IMAGE_PULL }}
run: |
# This script reads config.yaml, platforms.yaml and runners.yaml to generate a list of configurations to build and deploy
# It will;
# - check the source respository for the latest commit hash
# - check if the built image exists in our dockerhub registry
# - generate a list of configurations to build and deploy
#
# Here is an example of generate output;
# [{
# "source": {
# "repository": "hyperledger/besu",
# "ref": "main"
# },
# "build_script": "./besu/build.sh",
# "target": {
# "tag": "main",
# "repository": "ethpandaops/besu"
# },
# "platforms": [
# {
# "platform": "linux/amd64",
# "runner": "ubuntu-latest",
# "slug": "linux-amd64"
# },
# {
# "platform": "linux/arm64",
# "runner": "ARM64",
# "slug": "linux-arm64"
# }
# ]
# },{
# "source": {
# "repository": "sigp/lighthouse",
# "ref": "unstable"
# },
# "target": {
# "tag": "unstable",
# "repository": "ethpandaops/lighthouse",
# "dockerfile": "./lighthouse/Dockerfile"
# },
# "platforms": [
# {
# "platform": "linux/amd64",
# "runner": "ubuntu-latest",
# "slug": "linux-amd64"
# },
# {
# "platform": "linux/arm64",
# "runner": "ARM64",
# "slug": "linux-arm64"
# }
# ]
# }]
CONFIG_FILE="config.yaml"
PLATFORMS_FILE="platforms.yaml"
RUNNERS_FILE="runners.yaml"
# Create a temporary directory for storing intermediate results
TEMP_DIR=$(mktemp -d)
# Ensure the temporary directory is removed when the script exits
trap "rm -rf $TEMP_DIR" EXIT
process_commits() {
local LINE=$1
local SOURCE_REPOSITORY=$2
local SOURCE_REF=$3
local TARGET_REPOSITORY=$4
local TARGET_TAG=$5
local CLIENT="${TARGET_REPOSITORY#*/}"
local RESPONSE=$(curl -s -H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
"https://api.github.com/repos/${SOURCE_REPOSITORY}/commits/${SOURCE_REF}?per_page=1")
local COMMIT_HASH=$(echo "$RESPONSE" | jq -r '.sha' | cut -c1-7)
if [[ -z "$COMMIT_HASH" || "$COMMIT_HASH" == "null" ]]; then
# Log error but don't exit; just skip this configuration
echo "[LINE:$LINE] Error fetching commit hash for ${SOURCE_REPOSITORY}#${SOURCE_REF}, skipping."
return
fi
local configOutput="${TEMP_DIR}/${LINE}_commits.json"
touch $configOutput
echo "{\"line\": \"$LINE\", \"commit_hash\": \"$COMMIT_HASH\"}," >> $configOutput
}
process_image() {
local LINE=$1
local IMAGE=$2
local URL=$3
local imageOutput="${TEMP_DIR}/${LINE}_image.json"
touch $imageOutput
local exists=$(curl -s $URL | jq '.results | length > 0')
# check if exists == true
if [ "$exists" == "true" ]; then
exists=true
else
exists=false
fi
echo "{\"line\": \"$LINE\", \"image\": \"$IMAGE\", \"exists\": $exists}" >> $imageOutput
}
# Get commit hashes for each configuration in parallel
while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do
process_commits "$LINE" "$SOURCE_REPOSITORY" "$SOURCE_REF" "$TARGET_REPOSITORY" "$TARGET_TAG" &
done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE")
wait
# Initialize JSON arrays
COMMITS="["
# Concatenate results, ensuring files exist before attempting to read
for file in $TEMP_DIR/*_commits.json; do
if [ -f "$file" ]; then
COMMITS+=$(cat "$file")
fi
done
# Remove trailing commas and close JSON arrays
COMMITS="${COMMITS%,}]"
echo "Checking if images exist in dockerhub..."
while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do
# get the image commit hash from LINE
COMMIT_HASH=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash')
IMAGE_TAG="${TARGET_TAG}-${COMMIT_HASH}"
IMAGE="${TARGET_REPOSITORY}:${IMAGE_TAG}"
URL="https://hub.docker.com/v2/repositories/${TARGET_REPOSITORY}/tags?page_size=25&page=1&ordering=&name=${IMAGE_TAG}"
process_image $LINE $IMAGE $URL &
done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE")
wait
declare -A images
# Concatenate results, ensuring files exist before attempting to read
for file in $TEMP_DIR/*_image.json; do
if [ -f "$file" ]; then
LINE=$(cat "$file" | jq -r '.line')
IMAGE=$(cat "$file" | jq -r '.image')
EXISTS=$(cat "$file" | jq -r '.exists')
images[$IMAGE]=$EXISTS
fi
done
CONFIGS="configs=["
echo "Generating configuration files..."
while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do
# get the image commit hash from LINE
COMMIT_HASH=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash')
IMAGE_TAG="${TARGET_TAG}-${COMMIT_HASH}"
IMAGE="${TARGET_REPOSITORY}:${IMAGE_TAG}"
CLIENT="${TARGET_REPOSITORY#*/}"
if [ "${images[$IMAGE]}" == "false" ]; then
# Handle platforms and runners, ensuring output files are created even if empty
platforms=$(yq e ".$CLIENT[]" "$PLATFORMS_FILE")
platformsArr=""
for platform in $platforms; do
runner=$(yq e ".\"$platform\"" "$RUNNERS_FILE")
slug=$(echo "$platform" | tr '/' '-')
platformsArr+="{\\\"platform\\\": \\\"$platform\\\", \\\"runner\\\": \\\"$runner\\\", \\\"slug\\\": \\\"$slug\\\"},"
done
platformsArr="${platformsArr%,}"
# convert to string
platformsOutput="{\"platforms\": \"[$platformsArr]\"}"
CONFIGS+=$(echo "$(yq -r -o=json ".[${LINE}]" "$CONFIG_FILE" | jq --argjson plat "$platformsOutput" '. + $plat'),")
fi
done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE")
# Remove trailing commas and close JSON arrays
CONFIGS="${CONFIGS%,}]"
echo "CONFIGS: $CONFIGS"
echo $CONFIGS >> $GITHUB_OUTPUT
deploy:
needs: check
if: ${{ needs.check.outputs.configs != '[]' && needs.check.outputs.configs != '' }}
uses: ./.github/workflows/deploy.yml
strategy:
fail-fast: false
matrix:
config: ${{fromJson(needs.check.outputs.configs)}}
name: ${{ matrix.config.source.repository }}#${{ matrix.config.source.ref }} ${{ matrix.config.target.tag }}
with:
source_repository: ${{ matrix.config.source.repository }}
source_ref: ${{ matrix.config.source.ref }}
build_script: ${{ matrix.config.build_script }}
build_args: "${{ matrix.config.build_args }}"
target_tag: ${{ matrix.config.target.tag }}
target_repository: ${{ matrix.config.target.repository }}
target_dockerfile: ${{ matrix.config.target.dockerfile }}
platforms: ${{ matrix.config.platforms }}
DOCKER_USERNAME: "${{ vars.DOCKER_USERNAME }}"
GOPROXY: "${{ vars.GOPROXY }}"
secrets:
DOCKER_PASSWORD: "${{ secrets.DOCKER_PASSWORD }}"
MACOS_PASSWORD: "${{ secrets.MACOS_PASSWORD }}"
notify:
name: Discord Notification
runs-on: ubuntu-latest
needs:
- check
- deploy
if: cancelled() || failure()
steps:
- name: Notify
uses: nobrayner/discord-webhook@v1
with:
github-token: ${{ secrets.github_token }}
discord-webhook: ${{ secrets.DISCORD_WEBHOOK }}