scheduled.yml

name: Scheduled

on:
  schedule:
    - cron:  '45 * * * *'
  workflow_dispatch:

concurrency:
  group: "scheduled"
  cancel-in-progress: true

jobs:
  check:
    runs-on: ubuntu-latest
    outputs:
      configs: ${{ steps.repo_check.outputs.configs }}
    steps:
      - uses: actions/checkout@v4
      - uses: mikefarah/yq@v4.35.1
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ vars.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - id: repo_check
        env:
          CONCURRENT_IMAGE_PULL: ${{ vars.CONCURRENT_IMAGE_PULL }}
        run: |
          # This script reads config.yaml, platforms.yaml and runners.yaml to generate a list of configurations to build and deploy
          # It will;
          #    - check the source respository for the latest commit hash
          #    - check if the built image exists in our dockerhub registry
          #    - generate a list of configurations to build and deploy
          #
          # Here is an example of generate output;
          #  [{
          #    "source": {
          #      "repository": "hyperledger/besu",
          #      "ref": "main"
          #    },
          #    "build_script": "./besu/build.sh",
          #    "target": {
          #      "tag": "main",
          #      "repository": "ethpandaops/besu"
          #    },
          #    "platforms": [
          #      {
          #        "platform": "linux/amd64",
          #        "runner": "ubuntu-latest",
          #        "slug": "linux-amd64"
          #      },
          #      {
          #        "platform": "linux/arm64",
          #        "runner": "ARM64",
          #        "slug": "linux-arm64"
          #      }
          #    ]
          #  },{
          #    "source": {
          #      "repository": "sigp/lighthouse",
          #      "ref": "unstable"
          #    },
          #    "target": {
          #      "tag": "unstable",
          #      "repository": "ethpandaops/lighthouse",
          #      "dockerfile": "./lighthouse/Dockerfile"
          #    },
          #    "platforms": [
          #      {
          #        "platform": "linux/amd64",
          #        "runner": "ubuntu-latest",
          #        "slug": "linux-amd64"
          #      },
          #      {
          #        "platform": "linux/arm64",
          #        "runner": "ARM64",
          #        "slug": "linux-arm64"
          #      }
          #    ]
          #  }]

          CONFIG_FILE="config.yaml"
          PLATFORMS_FILE="platforms.yaml"
          RUNNERS_FILE="runners.yaml"

          # Create a temporary directory for storing intermediate results
          TEMP_DIR=$(mktemp -d)
          # Ensure the temporary directory is removed when the script exits
          trap "rm -rf $TEMP_DIR" EXIT

          process_commits() {
              local LINE=$1
              local SOURCE_REPOSITORY=$2
              local SOURCE_REF=$3
              local TARGET_REPOSITORY=$4
              local TARGET_TAG=$5

              local CLIENT="${TARGET_REPOSITORY#*/}"
              local RESPONSE=$(curl -s -H "Accept: application/vnd.github+json" \
                  -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
                  "https://api.github.com/repos/${SOURCE_REPOSITORY}/commits/${SOURCE_REF}?per_page=1")
              local COMMIT_HASH=$(echo "$RESPONSE" | jq -r '.sha' | cut -c1-7)

              if [[ -z "$COMMIT_HASH" || "$COMMIT_HASH" == "null" ]]; then
                  # Log error but don't exit; just skip this configuration
                  echo "[LINE:$LINE] Error fetching commit hash for ${SOURCE_REPOSITORY}#${SOURCE_REF}, skipping."
                  return
              fi

              local configOutput="${TEMP_DIR}/${LINE}_commits.json"
              touch $configOutput

              echo "{\"line\": \"$LINE\", \"commit_hash\": \"$COMMIT_HASH\"}," >> $configOutput
          }

          process_image() {
              local LINE=$1
              local IMAGE=$2
              local URL=$3

              local imageOutput="${TEMP_DIR}/${LINE}_image.json"
              touch $imageOutput
              local exists=$(curl -s $URL | jq '.results | length > 0')

              # check if exists == true
              if [ "$exists" == "true" ]; then
                exists=true
              else
                exists=false
              fi
              
              echo "{\"line\": \"$LINE\", \"image\": \"$IMAGE\", \"exists\": $exists}" >> $imageOutput
          }

          # Get commit hashes for each configuration in parallel
          while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do
              process_commits "$LINE" "$SOURCE_REPOSITORY" "$SOURCE_REF" "$TARGET_REPOSITORY" "$TARGET_TAG" &
          done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE")

          wait

          # Initialize JSON arrays
          COMMITS="["

          # Concatenate results, ensuring files exist before attempting to read
          for file in $TEMP_DIR/*_commits.json; do
              if [ -f "$file" ]; then
                  COMMITS+=$(cat "$file")
              fi
          done

          # Remove trailing commas and close JSON arrays
          COMMITS="${COMMITS%,}]"

          echo "Checking if images exist in dockerhub..."
          while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do
              # get the image commit hash from LINE
              COMMIT_HASH=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash')
              IMAGE_TAG="${TARGET_TAG}-${COMMIT_HASH}"
              IMAGE="${TARGET_REPOSITORY}:${IMAGE_TAG}"
              URL="https://hub.docker.com/v2/repositories/${TARGET_REPOSITORY}/tags?page_size=25&page=1&ordering=&name=${IMAGE_TAG}"
              process_image $LINE $IMAGE $URL &
          done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE")

          wait

          declare -A images

          # Concatenate results, ensuring files exist before attempting to read
          for file in $TEMP_DIR/*_image.json; do
              if [ -f "$file" ]; then
                  LINE=$(cat "$file" | jq -r '.line')
                  IMAGE=$(cat "$file" | jq -r '.image')
                  EXISTS=$(cat "$file" | jq -r '.exists')
                  images[$IMAGE]=$EXISTS
              fi
          done

          CONFIGS="configs=["

          echo "Generating configuration files..."
          while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do
              # get the image commit hash from LINE
              COMMIT_HASH=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash')
              IMAGE_TAG="${TARGET_TAG}-${COMMIT_HASH}"
              IMAGE="${TARGET_REPOSITORY}:${IMAGE_TAG}"
              CLIENT="${TARGET_REPOSITORY#*/}"

              if [ "${images[$IMAGE]}" == "false" ]; then
                # Handle platforms and runners, ensuring output files are created even if empty
                platforms=$(yq e ".$CLIENT[]" "$PLATFORMS_FILE")
                platformsArr=""

                for platform in $platforms; do
                  runner=$(yq e ".\"$platform\"" "$RUNNERS_FILE")
                  slug=$(echo "$platform" | tr '/' '-')
                  platformsArr+="{\\\"platform\\\": \\\"$platform\\\", \\\"runner\\\": \\\"$runner\\\", \\\"slug\\\": \\\"$slug\\\"},"
                done
                platformsArr="${platformsArr%,}"
                # convert to string
                platformsOutput="{\"platforms\": \"[$platformsArr]\"}"

                CONFIGS+=$(echo "$(yq -r -o=json ".[${LINE}]" "$CONFIG_FILE" | jq --argjson plat "$platformsOutput" '. + $plat'),")
              fi
          done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE")

          # Remove trailing commas and close JSON arrays
          CONFIGS="${CONFIGS%,}]"

          echo "CONFIGS: $CONFIGS"
          echo $CONFIGS >> $GITHUB_OUTPUT

  deploy:
    needs: check
    if: ${{ needs.check.outputs.configs != '[]' && needs.check.outputs.configs != '' }}
    uses: ./.github/workflows/deploy.yml
    strategy:
      fail-fast: false
      matrix:
        config: ${{fromJson(needs.check.outputs.configs)}}
    name: ${{ matrix.config.source.repository }}#${{ matrix.config.source.ref }} ${{ matrix.config.target.tag }}
    with:
      source_repository: ${{ matrix.config.source.repository }}
      source_ref: ${{ matrix.config.source.ref }}
      build_script: ${{ matrix.config.build_script }}
      build_args: "${{ matrix.config.build_args }}"
      target_tag: ${{ matrix.config.target.tag }}
      target_repository: ${{ matrix.config.target.repository }}
      target_dockerfile: ${{ matrix.config.target.dockerfile }}
      platforms: ${{ matrix.config.platforms }}
      DOCKER_USERNAME: "${{ vars.DOCKER_USERNAME }}"
      GOPROXY: "${{ vars.GOPROXY }}"
    secrets:
      DOCKER_PASSWORD: "${{ secrets.DOCKER_PASSWORD }}"
      MACOS_PASSWORD: "${{ secrets.MACOS_PASSWORD }}"

  notify:
    name: Discord Notification
    runs-on: ubuntu-latest
    needs:
      - check
      - deploy
    if: cancelled() || failure()
    steps:
      - name: Notify
        uses: nobrayner/discord-webhook@v1
        with:
          github-token: ${{ secrets.github_token }}
          discord-webhook: ${{ secrets.DISCORD_WEBHOOK }}