convert.yml

name: Convert & Check Images

on:
  schedule:
    # Do conversion every day at 00:03 clock UTC
    - cron: "3 0 * * *"
  workflow_dispatch:

env:
  CARGO_TERM_COLOR: always
  REGISTRY: ghcr.io
  ORGANIZATION: ${{ github.repository }}
  IMAGE_LIST_PATH: misc/top_images/image_list.txt
  FSCK_PATCH_PATH: misc/top_images/fsck.patch

jobs:
  nydusify-build:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v3
    - name: Setup Golang
      uses: actions/setup-go@v3
      with:
        go-version: ~1.18
    - name: Golang Cache
      uses: actions/cache@v3
      with:
        path: |
          ~/.cache/go-build
          ~/go/pkg/mod
        key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }}
        restore-keys: |
          ${{ runner.os }}-golang-
    - name: Build Contrib
      run: |
        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.51.2
        make -e DOCKER=false nydusify-release
    - name: Upload Nydusify
      uses: actions/upload-artifact@master
      with:
        name: nydusify-artifact
        path: contrib/nydusify/cmd/nydusify

  nydus-build:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v3
    - name: Rust Cache
      uses: Swatinem/rust-cache@v2.2.0
      with:
        cache-on-failure: true
        shared-key: nydus-build
    - name: Build Nydus
      run: |
        rustup component add rustfmt clippy
        make
    - name: Upload Nydus Binaries
      uses: actions/upload-artifact@master
      with:
        name: nydus-artifact
        path: |
          target/release/nydus-image
          target/release/nydusd

  fsck-erofs-build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Build fsck.erofs
        run: |
          sudo apt-get update && sudo apt-get install -y build-essential git autotools-dev automake libtool pkg-config uuid-dev liblz4-dev
          git clone https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git
          cd erofs-utils && git apply ../${{ env.FSCK_PATCH_PATH }} && ./autogen.sh && ./configure && make && cd ..
          sudo cp erofs-utils/fsck/fsck.erofs /usr/local/bin/
      - name: Upload fsck.erofs
        uses: actions/upload-artifact@master
        with:
          name: fsck-erofs-artifact
          path: |
            /usr/local/bin/fsck.erofs

  convert-zran:
    runs-on: ubuntu-latest
    needs: [nydusify-build, nydus-build, fsck-erofs-build]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Login ghcr registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Download Nydus
        uses: actions/download-artifact@master
        with:
          name: nydus-artifact
          path: /usr/local/bin
      - name: Download Nydusify
        uses: actions/download-artifact@master
        with:
          name: nydusify-artifact
          path: /usr/local/bin
      - name: Download fsck.erofs
        uses: actions/download-artifact@master
        with:
          name: fsck-erofs-artifact
          path: /usr/local/bin
      - name: Convert and check zran images
        run: |
          sudo chmod +x /usr/local/bin/nydus*
          sudo chmod +x /usr/local/bin/fsck.erofs
          sudo docker run -d --restart=always -p 5000:5000 registry
          sudo mkdir convert-zran
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            echo "converting $I:latest to $I:nydus-nightly-oci-ref"
            ghcr_repo=${{ env.REGISTRY }}/${{ env.ORGANIZATION }}

            # push oci image to ghcr/local for zran reference
            sudo docker pull $I:latest
            sudo docker tag $I:latest $ghcr_repo/$I
            sudo docker tag $I:latest localhost:5000/$I
            sudo DOCKER_CONFIG=$HOME/.docker docker push $ghcr_repo/$I
            sudo docker push localhost:5000/$I

            # for pre-built images
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --oci-ref \
                 --source $ghcr_repo/$I \
                 --target $ghcr_repo/$I:nydus-nightly-oci-ref \
                 --platform linux/amd64,linux/arm64

            # use local registry for speed
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --oci-ref \
                 --source localhost:5000/$I \
                 --target localhost:5000/$I:nydus-nightly-oci-ref \
                 --platform linux/amd64,linux/arm64 \
                 --output-json convert-zran/${I}.json

            # check zran image and referenced oci image
            sudo rm -rf ./tmp
            sudo DOCKER_CONFIG=$HOME/.docker nydusify check \
                --source localhost:5000/$I \
                --target localhost:5000/$I:nydus-nightly-oci-ref

            sudo fsck.erofs -d1 output/nydus_bootstrap
            sudo rm -rf ./output
          done
      - name: Save Nydusify Metric
        uses: actions/upload-artifact@v3
        with:
          name: convert-zran-metric
          path: convert-zran

  convert-native-v5:
    runs-on: ubuntu-latest
    needs: [nydusify-build, nydus-build]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Login ghcr registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Download Nydus
        uses: actions/download-artifact@master
        with:
          name: nydus-artifact
          path: /usr/local/bin
      - name: Download Nydusify
        uses: actions/download-artifact@master
        with:
          name: nydusify-artifact
          path: /usr/local/bin
      - name: Convert and check RAFS v5 images
        run: |
          sudo chmod +x /usr/local/bin/nydus*
          sudo docker run -d --restart=always -p 5000:5000 registry
          sudo mkdir convert-native-v5
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            echo "converting $I:latest to $I:nydus-nightly-v5"
            # for pre-built images
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --source $I:latest \
                 --target ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/$I:nydus-nightly-v5 \
                 --fs-version 5 \
                 --platform linux/amd64,linux/arm64

            # use local registry for speed
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --source $I:latest \
                 --target localhost:5000/$I:nydus-nightly-v5 \
                 --fs-version 5 \
                 --platform linux/amd64,linux/arm64 \
                 --output-json convert-native-v5/${I}.json

            sudo rm -rf ./tmp
            sudo DOCKER_CONFIG=$HOME/.docker nydusify check --source $I:latest \
                --target localhost:5000/$I:nydus-nightly-v5
          done
      - name: Save Nydusify Metric
        uses: actions/upload-artifact@v3
        with:
          name: convert-native-v5-metric
          path: convert-native-v5

  convert-native-v6:
    runs-on: ubuntu-latest
    needs: [nydusify-build, nydus-build, fsck-erofs-build]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Login ghcr registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Download Nydus
        uses: actions/download-artifact@master
        with:
          name: nydus-artifact
          path: /usr/local/bin
      - name: Download Nydusify
        uses: actions/download-artifact@master
        with:
          name: nydusify-artifact
          path: /usr/local/bin
      - name: Download fsck.erofs
        uses: actions/download-artifact@master
        with:
          name: fsck-erofs-artifact
          path: /usr/local/bin
      - name: Convert and check RAFS v6 images
        run: |
          sudo chmod +x /usr/local/bin/nydus*
          sudo chmod +x /usr/local/bin/fsck.erofs
          sudo docker run -d --restart=always -p 5000:5000 registry
          sudo mkdir convert-native-v6
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            echo "converting $I:latest to $I:nydus-nightly-v6"
            # for pre-built images
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --source $I:latest \
                 --target ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/$I:nydus-nightly-v6 \
                 --fs-version 6 \
                 --platform linux/amd64,linux/arm64

            # use local registry for speed
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --source $I:latest \
                 --target localhost:5000/$I:nydus-nightly-v6 \
                 --fs-version 6 \
                 --platform linux/amd64,linux/arm64 \
                 --output-json convert-native-v6/${I}.json

            sudo rm -rf ./tmp
            sudo DOCKER_CONFIG=$HOME/.docker nydusify check --source $I:latest \
                --target localhost:5000/$I:nydus-nightly-v6

            sudo fsck.erofs -d1 output/nydus_bootstrap
            sudo rm -rf ./output
          done
      - name: Save Nydusify Metric
        uses: actions/upload-artifact@v3
        with:
          name: convert-native-v6-metric
          path: convert-native-v6

  convert-metric:
    runs-on: ubuntu-latest
    needs: [convert-zran, convert-native-v5, convert-native-v6]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Download Zran Metric
        uses: actions/download-artifact@master
        with:
          name: convert-zran-metric
          path: convert-zran
      - name: Download V5 Metric
        uses: actions/download-artifact@master
        with:
          name: convert-native-v5-metric
          path: convert-native-v5
      - name: Download V6 Metric
        uses: actions/download-artifact@master
        with:
          name: convert-native-v6-metric
          path: convert-native-v6
      - name: Summary
        run: |
          echo "## Image Size(MB)" > $GITHUB_STEP_SUMMARY
          echo "|image name|nydus-zran|nydus-v5|nydus-v6|" >> $GITHUB_STEP_SUMMARY
          echo "|:---:|:--:|:-------:|:-------:|" >> $GITHUB_STEP_SUMMARY
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            zranSourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-zran/${I}.json) / 1048576")")
            zranTargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-zran/${I}.json) / 1048576")")
            v5SourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-native-v5/${I}.json) / 1048576")")
            v5TargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-native-v5/${I}.json) / 1048576")")
            v6SourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-native-v6/${I}.json) / 1048576")")
            v6TargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-native-v6/${I}.json) / 1048576")")
            echo "|${I}|${zranSourceImageSize}/${zranTargetImageSize}|${v5SourceImageSize}/${v5TargetImageSize}|${v6SourceImageSize}/${v6TargetImageSize}|" >> $GITHUB_STEP_SUMMARY
          done
          echo "## Conversion Time(ms)" >> $GITHUB_STEP_SUMMARY
          echo "|image name|nydus-zran|nydus-v5|nydus-v6|" >> $GITHUB_STEP_SUMMARY
          echo "|:---:|:--:|:-------:|:-------:|" >> $GITHUB_STEP_SUMMARY
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            zranConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-zran/${I}.json) / 1000000")")
            v5ConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-native-v5/${I}.json) / 1000000")")
            v6ConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-native-v6/${I}.json) / 1000000")")
            echo "|${I}|${zranConversionElapsed}|${v5ConversionElapsed}|${v6ConversionElapsed}|" >> $GITHUB_STEP_SUMMARY
          done
      - uses: geekyeggo/delete-artifact@v2
        with:
          name: '*'