From d433bba50c401556720dc7d748cb2665c6ee02f6 Mon Sep 17 00:00:00 2001 From: gtxaspec Date: Wed, 27 Nov 2024 03:59:10 -0800 Subject: [PATCH] workflow: update firmware-module --- .github/workflows/firmware-module.yaml | 159 +++++++++++++++---------- 1 file changed, 99 insertions(+), 60 deletions(-) diff --git a/.github/workflows/firmware-module.yaml b/.github/workflows/firmware-module.yaml index 29236ac79..65f857a60 100644 --- a/.github/workflows/firmware-module.yaml +++ b/.github/workflows/firmware-module.yaml @@ -1,15 +1,15 @@ name: firmware-module on: schedule: - # 03:15 PST / Nightly - - cron: '15 10 * * *' + # 03:30 PST / Nightly + - cron: '30 10 * * *' workflow_dispatch: inputs: tg_disabled: type: boolean description: 'Disable Telegram notifications' required: false - default: false + default: true tg_scratch: type: boolean description: 'Use TG scratch channel' @@ -25,7 +25,7 @@ on: description: 'Only build Tea Cup profile' required: false default: false - debug_only: + debug_enabled: type: boolean description: 'Debug: Generate dummy image files' required: false @@ -33,14 +33,15 @@ on: env: TAG_NAME: module - UPD_TAG_NAME: module_update TERM: linux BR2_DL_DIR: ~/dl TG_TOKEN: ${{secrets.TELEGRAM_TOKEN_BOT_THINGINO}} - TG_CHANNEL: ${{secrets.TELEGRAM_CHANNEL_THINGINO_BUILD}} + TG_CHANNEL: -1002083893006_14394 + TG_TOPIC: 14394 TG_CHANNEL_SCRATCH: ${{secrets.TELEGRAM_CHANNEL_THINGINO_SCRATCH}} TG_OPTIONS: -s FORCE_UNSAFE_CONFIGURE: 1 + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: notify-begin: @@ -49,16 +50,25 @@ jobs: start_time: ${{ steps.set_output.outputs.time }} tg_disabled: ${{ steps.set_env.outputs.tg_disabled }} steps: - - name: Save workflow start time + - name: Set timezone + run: | + sudo timedatectl set-timezone "America/Los_Angeles" + + - name: Save workflow start time to ENV id: set_output run: echo "time=$(date +%s)" >> $GITHUB_OUTPUT - - name: Setup Notify Environment + - name: Configure Environment Variables run: | echo "TG_DISABLED=${{ github.event.inputs.tg_disabled || 'false' }}" >> $GITHUB_ENV echo "tg_disabled=${{ github.event.inputs.tg_disabled || 'false' }}" >> $GITHUB_OUTPUT - - name: Send build start notifcation + - name: Setup Notification Channel + if: ${{ github.event.inputs.tg_scratch == 'true' }} + run: | + echo "TG_TOPIC=" >> $GITHUB_ENV + + - name: Send build start notifcation via Telegram if: env.TG_DISABLED == 'false' run: | if [[ "${{ github.event.inputs.tg_scratch }}" == 'true' ]]; then @@ -67,7 +77,7 @@ jobs: TG_MSG="Firmware\\-module build started:\nJob: [${{ github.run_id }}](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n\n" TG_ICON="\xF0\x9F\x9A\xA6 GitHub Actions" TG_HEADER=$(echo -e ${TG_MSG}${TG_ICON}) - HTTP=$(curl ${TG_OPTIONS} -H "Content-Type: multipart/form-data" -X POST https://api.telegram.org/bot${TG_TOKEN}/sendMessage -F parse_mode=MarkdownV2 -F chat_id=${TG_CHANNEL} -F text="${TG_HEADER}" -F disable_web_page_preview=true) + HTTP=$(curl ${TG_OPTIONS} -H "Content-Type: multipart/form-data" -X POST https://api.telegram.org/bot${TG_TOKEN}/sendMessage -F parse_mode=MarkdownV2 -F message_thread_id=${TG_TOPIC} -F chat_id=${TG_CHANNEL} -F text="${TG_HEADER}" -F disable_web_page_preview=true) echo Telegram response: ${HTTP} generate-matrix: @@ -75,16 +85,20 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - name: setup env test + - name: Set timezone + run: | + sudo timedatectl set-timezone "America/Los_Angeles" + + - name: Configure GH workspace run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: Checkout source + - name: Checkout repository uses: actions/checkout@v4 with: ref: "master" fetch-depth: "1" - - name: Generate matrix + - name: Generate device matrix id: set-matrix run: | if [[ "${{ github.event.inputs.teacup_only }}" == 'true' ]]; then @@ -104,6 +118,10 @@ jobs: name: ${{ matrix.thingino-version }} needs: [generate-matrix, notify-begin] runs-on: ubuntu-latest + outputs: + # Feed the hash of the commit we are building into the notify-completion release-notes step + GIT_HASH: ${{ steps.env.outputs.git_hash }} + TAG_NAME: ${{ steps.env.outputs.tag_name }} defaults: run: shell: bash @@ -112,36 +130,42 @@ jobs: strategy: fail-fast: false matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}} - steps: - - name: Update APT sources + - name: Update package manager sources run: | apt-get update - name: Install build dependencies run: | - apt-get install -y --no-install-recommends --no-install-suggests build-essential bc ccache cmake cpio curl ca-certificates file git make gawk procps rsync u-boot-tools unzip wget + apt-get install -y --no-install-recommends --no-install-suggests build-essential bc cmake cpio curl ca-certificates file git make gawk jq procps rsync tzdata u-boot-tools unzip wget + + - name: Set timezone + run: | + ln -sf /usr/share/zoneinfo/America/Los_Angeles /etc/localtime + echo "America/Los_Angeles" > /etc/timezone + DEBIAN_FRONTEND=noninteractive dpkg-reconfigure -f noninteractive tzdata - name: Setup gh workspace in container run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: Checkout source + - name: Checkout repository source uses: actions/checkout@v4 with: submodules: 'true' ref: "master" fetch-depth: "1" - - name: Setup Environment - id: date + - name: Configure Environment Variables + id: env run: | echo "WEEK_NUMBER=$(date +%U)" >> $GITHUB_ENV echo "CURRENT_YEAR=$(date +%Y)" >> $GITHUB_ENV - export GIT_HASH=$(git rev-parse --short ${GITHUB_SHA}) - export GIT_BRANCH=${GITHUB_REF_NAME} + echo "GIT_HASH=$(git rev-parse --short ${GITHUB_SHA})" >> $GITHUB_ENV + echo "GIT_BRANCH=${GITHUB_REF_NAME}" >> $GITHUB_ENV echo "GIT_HASH=${GIT_HASH}" >> ${GITHUB_ENV} echo "GIT_BRANCH=${GIT_BRANCH}" >> ${GITHUB_ENV} echo "TG_DISABLED=${{ github.event.inputs.tg_disabled || 'false' }}" >> $GITHUB_ENV + echo "GIT_HASH=$(git rev-parse --short ${GITHUB_SHA})" >> $GITHUB_OUTPUT - name: Setup cache directories run: | @@ -153,9 +177,12 @@ jobs: if: always() with: path: ~/.ccache - key: ${{ runner.os }}-ccache-${{ matrix.thingino-version }}-${{ env.CURRENT_YEAR }}-week-${{ env.WEEK_NUMBER }} + key: ${{ runner.os }}-ccache-${{ matrix.thingino-version }}-${{ env.CURRENT_YEAR }}-week-${{ env.WEEK_NUMBER }}-${{ github.run_id }} restore-keys: | + ${{ runner.os }}-ccache-${{ matrix.thingino-version }}-${{ env.CURRENT_YEAR }}-week-${{ env.WEEK_NUMBER }} ${{ runner.os }}-ccache-${{ matrix.thingino-version }}-${{ env.CURRENT_YEAR }}-week- + ${{ runner.os }}-ccache-${{ matrix.thingino-version }}- + ${{ runner.os }}-ccache- ${{ runner.os }}-ccache-shared-v1-${{ env.CURRENT_YEAR }}-week-${{ env.WEEK_NUMBER }} - name: Restore Buildroot DL cache @@ -166,28 +193,47 @@ jobs: key: ${{ runner.os }}-dl-shared-v1-${{ env.CURRENT_YEAR }}-week-${{ env.WEEK_NUMBER }} - name: Build firmware + if: ${{ github.event.inputs.debug_enabled != 'true' }} run: | BOARD=${{ matrix.thingino-version }} make fast TIME=$(date -d @${SECONDS} +%M:%S) echo "TIME=${TIME}" >> ${GITHUB_ENV} - - name: Generate build graphs + - name: Generate debug dummy firmware for workflow testing + if: ${{ github.event.inputs.debug_enabled == 'true' }} + run: | + DYNAMIC_PART="${{ matrix.thingino-version }}" + mkdir -p ${HOME}/output/${DYNAMIC_PART}/images/ + echo "debug" > ${HOME}/output/${DYNAMIC_PART}/images/thingino-${DYNAMIC_PART}.bin + echo "debug uboot" > ${HOME}/output/${DYNAMIC_PART}/images/u-boot-lzo-with-spl.bin + echo "debug kernel" > ${HOME}/output/${DYNAMIC_PART}/images/uImage + echo "debug rootfs" > ${HOME}/output/${DYNAMIC_PART}/images/rootfs.tar + echo "debug sha" > ${HOME}/output/${DYNAMIC_PART}/images/thingino-${DYNAMIC_PART}.bin.sha256sum + echo "debug" > ${HOME}/output/${DYNAMIC_PART}/images/thingino-${DYNAMIC_PART}-update.bin + echo "debug sha" > ${HOME}/output/${DYNAMIC_PART}/images/thingino-${DYNAMIC_PART}-update.bin.sha256sum + + - name: Generate build time graphs if: ${{ github.event.inputs.graph_enabled == 'true' }} run: | apt-get install -y --no-install-recommends --no-install-suggests python3-numpy python3-matplotlib BOARD=${{ matrix.thingino-version }} make br-graph-build - - name: Find built firmware images + - name: Locate Built Firmware Artifacts run: | DYNAMIC_PART="${{ matrix.thingino-version }}" FULL_FW=$(find ${HOME}/output/${DYNAMIC_PART}*/images/ -name "thingino-${DYNAMIC_PART}.bin" ! -name "*update.bin" | head -n 1) + FULL_FW_UBOOT=$(find ${HOME}/output/${DYNAMIC_PART}*/images/ -name "u-boot-lzo-with-spl.bin" | head -n 1) + FULL_FW_KERNEL=$(find ${HOME}/output/${DYNAMIC_PART}*/images/ -name "uImage" | head -n 1) + FULL_FW_ROOTFS=$(find ${HOME}/output/${DYNAMIC_PART}*/images/ -name "rootfs.tar" | head -n 1) UPDATE_FW=$(find ${HOME}/output/${DYNAMIC_PART}*/images/ -name "thingino-${DYNAMIC_PART}-update.bin" | head -n 1) echo "FULL_FW: $FULL_FW" - echo "UPDATE_FW: $UPDATE_FW" - if [[ -n "$FULL_FW" && -n "$UPDATE_FW" ]]; then + if [[ -n "$FULL_FW" ]]; then echo "FULL_FW=${FULL_FW}" >> ${GITHUB_ENV} - echo "UPDATE_FW=${UPDATE_FW}" >> ${GITHUB_ENV} + echo "FULL_FW_UBOOT=${FULL_FW_UBOOT}" >> ${GITHUB_ENV} + echo "FULL_FW_KERNEL=${FULL_FW_KERNEL}" >> ${GITHUB_ENV} + echo "FULL_FW_ROOTFS=${FULL_FW_ROOTFS}" >> ${GITHUB_ENV} echo "FULL_FW_SHA=${FULL_FW}.sha256sum" >> ${GITHUB_ENV} + echo "UPDATE_FW=${UPDATE_FW}" >> ${GITHUB_ENV} echo "UPDATE_FW_SHA=${UPDATE_FW}.sha256sum" >> ${GITHUB_ENV} else echo "Matching .bin files not found." @@ -202,21 +248,24 @@ jobs: path: | ~/output/${{ matrix.thingino-version }}/graphs/build.hist-duration.pdf - - name: Upload full firmware artifact + - name: Upload update firmware artifacts uses: actions/upload-artifact@v4 with: - name: thingino-${{ matrix.thingino-version }}-full-firmware + name: thingino-${{ matrix.thingino-version }}-update-firmware path: | - ${{ env.FULL_FW }} - ${{ env.FULL_FW_SHA }} + ${{ env.UPDATE_FW }} + ${{ env.UPDATE_FW_SHA }} - - name: Upload update firmware artifact + - name: Upload full firmware artifacts uses: actions/upload-artifact@v4 with: - name: thingino-${{ matrix.thingino-version }}-update-firmware + name: thingino-${{ matrix.thingino-version }}-full-firmware path: | - ${{ env.UPDATE_FW }} - ${{ env.UPDATE_FW_SHA }} + ${{ env.FULL_FW }} + ${{ env.FULL_FW_UBOOT }} + ${{ env.FULL_FW_KERNEL }} + ${{ env.FULL_FW_ROOTFS }} + ${{ env.FULL_FW_SHA }} - name: Upload full firmware to release uses: softprops/action-gh-release@master @@ -227,35 +276,20 @@ jobs: ${{ env.FULL_FW }} ${{ env.FULL_FW_SHA }} - - name: Upload update firmware to release - uses: softprops/action-gh-release@master - with: - tag_name: ${{ env.UPD_TAG_NAME }} - make_latest: false - files: | - ${{ env.UPDATE_FW }} - ${{ env.UPDATE_FW_SHA }} - - name: Send firmware completion notifications with binaries - if: ${{ env.TG_DISABLED == 'false' && (env.FULL_FW || env.UPDATE_FW) }} + if: ${{ env.TG_DISABLED == 'false' && (env.FULL_FW) }} run: | if [[ "${{ github.event.inputs.tg_scratch }}" == 'true' ]]; then TG_CHANNEL=${{ env.TG_CHANNEL_SCRATCH }} fi TG_ESCAPED_TAG_NAME=$(echo "${TAG_NAME}" | sed 's/_/\\_/g') if [ -n "${{ env.FULL_FW }}" ]; then - TG_MSG="Commit: [${GIT_HASH}](https://github.com/${GITHUB_REPOSITORY}/commit/${GIT_HASH})\nBranch: [${GIT_BRANCH}](https://github.com/${GITHUB_REPOSITORY}/tree/${GIT_BRANCH})\nTag: [${TG_ESCAPED_TAG_NAME}](https://github.com/${GITHUB_REPOSITORY}/releases/tag/${TAG_NAME})\nTime: ${TIME}\nJob: [${{ github.run_id }}](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n\n" - TG_HEADER=$(echo -e "${TG_MSG}\xE2\x9C\x85 GitHub Actions") - HTTP_FULL=$(curl ${TG_OPTIONS} -H "Content-Type: multipart/form-data" -X POST https://api.telegram.org/bot${TG_TOKEN}/sendDocument -F parse_mode=MarkdownV2 -F chat_id=${TG_CHANNEL} -F caption="${TG_HEADER}" -F document=@${FULL_FW} -F disable_web_page_preview=true) - echo "Telegram response Full Firmware: $HTTP_FULL" - fi + ESCAPED_TAG_NAME=$(echo "${TAG_NAME}" | sed 's/-/\\-/g') - TG_ESCAPED_UPD_TAG_NAME=$(echo "${UPD_TAG_NAME}" | sed 's/_/\\_/g') - if [ -n "${{ env.UPDATE_FW }}" ]; then - TG_MSG="Commit: [${GIT_HASH}](https://github.com/${GITHUB_REPOSITORY}/commit/${GIT_HASH})\nBranch: [${GIT_BRANCH}](https://github.com/${GITHUB_REPOSITORY}/tree/${GIT_BRANCH})\nTag: [${TG_ESCAPED_UPD_TAG_NAME}](https://github.com/${GITHUB_REPOSITORY}/releases/tag/firmware\\_update)\nTime: ${TIME}\nJob: [${{ github.run_id }}](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n\n" + TG_MSG="Commit: [${GIT_HASH}](https://github.com/${GITHUB_REPOSITORY}/commit/${GIT_HASH})\nBranch: [${GIT_BRANCH}](https://github.com/${GITHUB_REPOSITORY}/tree/${GIT_BRANCH})\nTag: [${ESCAPED_TAG_NAME}](https://github.com/${GITHUB_REPOSITORY}/releases/tag/${ESCAPED_TAG_NAME})\nBuild Time: ${TIME}\nJob: [${{ github.run_id }}](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n\n" TG_HEADER=$(echo -e "${TG_MSG}\xE2\x9C\x85 GitHub Actions") - HTTP_UPDATE=$(curl ${TG_OPTIONS} -H "Content-Type: multipart/form-data" -X POST https://api.telegram.org/bot${TG_TOKEN}/sendDocument -F parse_mode=MarkdownV2 -F chat_id=${TG_CHANNEL} -F caption="${TG_HEADER}" -F document=@${UPDATE_FW} -F disable_web_page_preview=true) - echo "Telegram response Update Firmware: $HTTP_UPDATE" + HTTP_FULL=$(curl ${TG_OPTIONS} -H "Content-Type: multipart/form-data" -X POST https://api.telegram.org/bot${TG_TOKEN}/sendDocument -F parse_mode=MarkdownV2 -F message_thread_id=${TG_TOPIC} -F chat_id=${TG_CHANNEL} -F caption="${TG_HEADER}" -F document=@${FULL_FW} -F disable_web_page_preview=true) + echo "Telegram response Full Firmware: $HTTP_FULL" fi - name: Send error notification @@ -265,12 +299,11 @@ jobs: TG_CHANNEL=${{ env.TG_CHANNEL_SCRATCH }} fi TG_ESCAPED_VERSION=$(echo "${{ matrix.thingino-version }}" | sed 's/_/\\_/g') - TG_ESCAPED_TAG_NAME=$(echo "${TAG_NAME}" | sed 's/_/\\_/g') TG_WARN="Error: ${TG_ESCAPED_VERSION}\n" - TG_MSG="Commit: [${GIT_HASH}](https://github.com/${GITHUB_REPOSITORY}/commit/${GIT_HASH})\nBranch: [${GIT_BRANCH}](https://github.com/${GITHUB_REPOSITORY}/tree/${GIT_BRANCH})\nTag: [${TG_ESCAPED_TAG_NAME}](https://github.com/${GITHUB_REPOSITORY}/releases/tag/${TAG_NAME})\nJob: [${{ github.run_id }}](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n\n" + TG_MSG="Commit: [${GIT_HASH}](https://github.com/${GITHUB_REPOSITORY}/commit/${GIT_HASH})\nBranch: [${GIT_BRANCH}](https://github.com/${GITHUB_REPOSITORY}/tree/${GIT_BRANCH})\nTag: [${TAG_NAME}](https://github.com/${GITHUB_REPOSITORY}/releases/tag/${TAG_NAME})\nJob: [${{ github.run_id }}](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n\n" TG_ICON="\xE2\x9A\xA0 GitHub Actions" TG_HEADER=$(echo -e ${TG_WARN}${TG_MSG}${TG_ICON}) - HTTP=$(curl ${TG_OPTIONS} -H "Content-Type: multipart/form-data" -X POST https://api.telegram.org/bot${TG_TOKEN}/sendMessage -F parse_mode=MarkdownV2 -F chat_id=${TG_CHANNEL} -F text="${TG_HEADER}" -F disable_web_page_preview=true) + HTTP=$(curl ${TG_OPTIONS} -H "Content-Type: multipart/form-data" -X POST https://api.telegram.org/bot${TG_TOKEN}/sendMessage -F parse_mode=MarkdownV2 -F message_thread_id=${TG_TOPIC} -F chat_id=${TG_CHANNEL} -F text="${TG_HEADER}" -F disable_web_page_preview=true) echo Telegram response: ${HTTP} notify-completion: @@ -278,9 +311,15 @@ jobs: runs-on: ubuntu-latest if: always() steps: - - name: Setup Environment + - name: Set timezone + run: | + sudo timedatectl set-timezone "America/Los_Angeles" + + - name: Configure Environment run: | echo "TG_DISABLED=${{ github.event.inputs.tg_disabled || 'false' }}" >> $GITHUB_ENV + echo "GIT_HASH=${{ needs.buildroot.outputs.git_hash }}" >> $GITHUB_ENV + echo "TAG_NAME=${{ needs.buildroot.outputs.tag_name }}" >> $GITHUB_ENV - name: Send completion summary if: ${{ env.TG_DISABLED == 'false' }} @@ -296,5 +335,5 @@ jobs: TG_MSG="Firmware\\-module build completed:\nTotal elapsed time: ${ELAPSED_MIN}m ${ELAPSED_SEC}s\nJob: [${{ github.run_id }}](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n\n" TG_ICON="\xF0\x9F\x9A\xA9 GitHub Actions" TG_HEADER=$(echo -e ${TG_MSG}${TG_ICON}) - HTTP=$(curl ${TG_OPTIONS} -H "Content-Type: multipart/form-data" -X POST https://api.telegram.org/bot${TG_TOKEN}/sendMessage -F parse_mode=MarkdownV2 -F chat_id=${TG_CHANNEL} -F text="${TG_HEADER}" -F disable_web_page_preview=true) + HTTP=$(curl ${TG_OPTIONS} -H "Content-Type: multipart/form-data" -X POST https://api.telegram.org/bot${TG_TOKEN}/sendMessage -F parse_mode=MarkdownV2 -F message_thread_id=${TG_TOPIC} -F chat_id=${TG_CHANNEL} -F text="${TG_HEADER}" -F disable_web_page_preview=true) echo Telegram response: ${HTTP}