generate-mirror.yml

# /********************************************************************************
# * Copyright (c) 2022 Contributors to the Eclipse Foundation
# *
# * See the NOTICE file(s) distributed with this work for additional
# * information regarding copyright ownership.
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0
# *
# * SPDX-License-Identifier: Apache-2.0
# ********************************************************************************/
#
# Download all the sources of the Leda / Yocto packages, create a tar ball 
# in the build/downloads and upload the *.tar.gz files to the remote downloads mirror.
# - Runs every Sunday
# - Archives the source from upstream repositories
# - Synchronizes to remote mirror on Azure Storage
# - Optimizes build time 
# - Prevents slow repositories from slowing down the build
# - Prevents build failures due to offline upstream repositories or temporary unavailability
# - Decreases load on upstream repositories
# - Typical size of mirror is about 15GB
#
# Slow repositories are:
# - sourceware.org (binutils, cross-localedef)
# - git.yoctoproject.org (linux-yocto)
#
name: Generate Yocto Mirror

on:
  workflow_dispatch:
  schedule:
    - cron: '35 13 * * SUN'

jobs:
  generate-mirror:
    runs-on: ubuntu-22.04
    steps:
    - name: Checkout
      uses: actions/checkout@v3
    - name: Install tools
      run: |
        sudo apt-get update
        sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool tmux mc skopeo fdisk ruby-full jq libvirt-clients libvirt-daemon-system qemu-system-x86 qemu-system-arm qemu-kvm squashfs-tools rauc python3-newt ca-certificates curl gnupg lsb-release
    - name: Install kas
      run: sudo pip3 install kas
    - name: Fetch all
      continue-on-error: true
      run: |
        KAS_MACHINE=qemux86-64 kas shell -c 'bitbake --continue --runall=fetch sdv-image-all' kas/generate-mirror.yaml
        KAS_MACHINE=qemuarm64 kas shell -c 'bitbake --continue --runall=fetch sdv-image-all' kas/generate-mirror.yaml
        KAS_MACHINE=raspberrypi4-64 kas shell -c 'bitbake --continue --runall=fetch sdv-image-all' kas/generate-mirror.yaml
    - name: List archives
      continue-on-error: true
      run: ls -al build/downloads/
    - uses: bacongobbler/azure-blob-storage-upload@v3.0.0
      if: ${{ github.repository_owner == 'SoftwareDefinedVehicle' }}
      with:
        source_dir: 'build/downloads/'
        container_name: 'downloads'
        extra_args: '--pattern *.tar.*[!done]'
        overwrite: 'sync'
        connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}