Remove actions/cache to improve workflow speed / Refactoring docker-b…

…uild workflows (#957)

* 🔥 Remove cache-docker-layers step

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

* ♻️ Add internal action/docker-build

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

✏️ Remove blankline

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

🐛 Fix uses statement

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

✏️ Fix typo

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

✏️ Fix typo

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

✨ Add logs

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

💚 Fix bug on if-else

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

* ♻️ Use action/docker-build for other images

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

* ♻️ Add option to specify platforms. Use docker-build in other images

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

* ♻️ Apply docker-build action to gateway-filter

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

.github/actions/docker-build/action.yaml

@@ -0,0 +1,115 @@
+name: 'Build Docker images'
+description: 'Build Docker images and publish them'
+inputs:
+  target:
+    description: 'build target'
+    required: true
+    default: 'base'
+  builder:
+    description: 'buildx builder name'
+    required: true
+    default: ''
+  platforms:
+    description: 'if it is specified, specified platforms will be used.'
+    required: false
+    default: ''
+outputs:
+  IMAGE_NAME:
+    description: "image name"
+    value: ${{ steps.image_name.outputs.IMAGE_NAME }}
+  ALTER_IMAGE_NAME:
+    description: "alter image name"
+    value: ${{ steps.image_name.outputs.ALTER_IMAGE_NAME }}
+  PRIMARY_TAG:
+    description: "primary tag"
+    value: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
+  PLATFORMS:
+    description: "target platforms"
+    value: ${{ steps.determine_platforms.outputs.PLATFORMS }}
+  EXTRA_TAGS:
+    description: "extra tags"
+    value: ${{ steps.add_extra_tags.outputs.EXTRA_TAGS }}
+runs:
+  using: "composite"
+  steps:
+    - name: Image name
+      shell: bash
+      id: image_name
+      run: |
+        image_name=`make docker/name/${TARGET}`
+        alter_org=`make docker/name/org/alter`
+        alter_image_name=`make ORG="${alter_org}" docker/name/${TARGET}`
+        echo "IMAGE_NAME is: ${image_name}"
+        echo "ALTER_IMAGE_NAME is: ${alter_image_name}"
+        echo "::set-output name=IMAGE_NAME::${image_name}"
+        echo "::set-output name=ALTER_IMAGE_NAME::${alter_image_name}"
+      env:
+        TARGET: ${{ inputs.target }}
+    - name: Determine tag name
+      shell: bash
+      id: determine_tag_name
+      run: |
+        if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then
+          tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'`
+          primary_tag="${tag_name}"
+        elif [ "$GITHUB_REF" = "refs/heads/master" ]; then
+          echo "nightly" > versions/VALD_VERSION
+          primary_tag="nightly"
+        elif [ "${{ github.event_name }}" = "pull_request" ]; then
+          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+          echo "PR-${pr_num}" > versions/VALD_VERSION
+          primary_tag="pr-${pr_num}"
+        else
+          primary_tag="unknown"
+        fi
+        echo "PRIMARY_TAG is determined: ${primary_tag}"
+        echo "::set-output name=PRIMARY_TAG::${primary_tag}"
+    - name: Determine platforms
+      shell: bash
+      id: determine_platforms
+      run: |
+        if [ "${TARGET_PLATFORMS}" = "" ]; then
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            platforms="linux/amd64"
+          else
+            platforms=`make docker/platforms`
+          fi
+        else
+          platforms="${TARGET_PLATFORMS}"
+        fi
+        echo "PLATFORMS is determined: ${platforms}"
+        echo "::set-output name=PLATFORMS::${platforms}"
+      env:
+        TARGET_PLATFORMS: ${{ inputs.platforms }}
+    - name: Add extra tags
+      shell: bash
+      id: add_extra_tags
+      run: |
+        extra_tags="-t ${ALTER_IMAGE_NAME}:${PRIMARY_TAG}"
+        if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then
+          latest_tags="-t ${IMAGE_NAME}:latest -t ${ALTER_IMAGE_NAME}:latest"
+          extra_tags="${extra_tags} ${latest_tags}"
+        fi
+        echo "EXTRA_TAGS is determined: ${extra_tags}"
+        echo "::set-output name=EXTRA_TAGS::${extra_tags}"
+      env:
+        IMAGE_NAME: ${{ steps.image_name.outputs.IMAGE_NAME }}
+        ALTER_IMAGE_NAME: ${{ steps.image_name.outputs.ALTER_IMAGE_NAME }}
+        PRIMARY_TAG: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
+    - name: Build and Push
+      shell: bash
+      id: build_and_push
+      run: |
+        make \
+          DOCKER="docker buildx" \
+          DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${LABEL_OPTS} ${EXTRA_TAGS} --push" \
+          TAG="${PRIMARY_TAG}" \
+          docker/build/${TARGET}
+      env:
+        TARGET: ${{ inputs.target }}
+        DOCKER_BUILDKIT: "1"
+        PLATFORMS: ${{ steps.determine_platforms.outputs.PLATFORMS }}
+        BUILDER: ${{ inputs.builder }}
+        LABEL_OPTS: "--label org.opencontainers.image.url=${{ github.event.repository.html_url }} --label org.opencontainers.image.source=${{ github.event.repository.html_url }} --label org.opencontainers.image.revision=${{ github.sha }}"
+        EXTRA_TAGS: ${{ steps.add_extra_tags.outputs.EXTRA_TAGS }}
+        PRIMARY_TAG: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}

.github/workflows/dockers-agent-ngt-image.yml

@@ -67,13 +67,6 @@ jobs:
         uses: docker/setup-buildx-action@v1
         with:
           buildkitd-flags: "--debug"
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-vald-agent-ngt-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-vald-agent-ngt-
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
@@ -85,79 +78,26 @@ jobs:
           registry: ghcr.io
           username: ${{ secrets.PACKAGE_USER }}
           password: ${{ secrets.PACKAGE_TOKEN }}
-      - name: Image name
-        id: image_name
-        run: |
-          image_name=`make docker/name/agent-ngt`
-          alter_org=`make docker/name/org/alter`
-          alter_image_name=`make ORG="${alter_org}" docker/name/agent-ngt`
-          base_platforms=`make docker/platforms`
-          echo "IMAGE_NAME=${image_name}" >> $GITHUB_ENV
-          echo "ALTER_IMAGE_NAME=${alter_image_name}" >> $GITHUB_ENV
-          echo "::set-output name=IMAGE_NAME::${image_name}"
-          echo "::set-output name=BASE_PLATFORMS::${base_platforms}"
-      - name: Determine tag name (master)
-        if: github.ref == 'refs/heads/master'
-        run: |
-          echo "nightly" > versions/VALD_VERSION
-          echo "PRIMARY_TAG=nightly" >> $GITHUB_ENV
-          echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
-        env:
-          PLATFORMS: ${{ steps.image_name.outputs.BASE_PLATFORMS }}
-      - name: Determine tag name (pull request)
-        if: github.event_name == 'pull_request'
-        run: |
-          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
-          echo "PR-${pr_num}" > versions/VALD_VERSION
-          echo "PRIMARY_TAG=pr-${pr_num}" >> $GITHUB_ENV
-          echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
-        env:
-          PLATFORMS: linux/amd64
-      - name: Determine tag name (tags)
-        if: startsWith( github.ref, 'refs/tags/')
-        id: determine_tag
-        run: |
-          tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'`
-          echo "::set-output name=TAG_NAME::${tag_name}"
-          echo "PRIMARY_TAG=${tag_name}" >> $GITHUB_ENV
-          echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
-        env:
-          PLATFORMS: ${{ steps.image_name.outputs.BASE_PLATFORMS }}
-      - name: Add extra tags
-        run: |
-          EXTRA_TAGS="-t ${ALTER_IMAGE_NAME}:${PRIMARY_TAG}"
-          echo "EXTRA_TAGS=${EXTRA_TAGS}" >> $GITHUB_ENV
-      - name: Add latest tags
-        if: startsWith( github.ref, 'refs/tags/')
-        run: |
-          LATEST_TAGS="-t ${IMAGE_NAME}:latest -t ${ALTER_IMAGE_NAME}:latest"
-          echo "LATEST_TAGS=${LATEST_TAGS}" >> $GITHUB_ENV
-      - name: Build and Push
-        run: |
-          make \
-            DOCKER="docker buildx" \
-            DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${CACHE_OPTS} ${LABEL_OPTS} ${EXTRA_TAGS} ${LATEST_TAGS} --push" \
-            TAG="${PRIMARY_TAG}" \
-            docker/build/agent-ngt
-        env:
-          DOCKER_BUILDKIT: 1
-          BUILDER: ${{ steps.buildx.outputs.name }}
-          CACHE_OPTS: "--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache"
-          LABEL_OPTS: "--label org.opencontainers.image.url=${{ github.event.repository.html_url }} --label org.opencontainers.image.source=${{ github.event.repository.html_url }} --label org.opencontainers.image.revision=${{ github.sha }}"
+      - name: Build and Publish
+        id: build_and_publish
+        uses: ./.github/actions/docker-build
+        with:
+          target: agent-ngt
+          builder: ${{ steps.buildx.outputs.name }}
       - name: Initialize CodeQL
         if: startsWith( github.ref, 'refs/tags/')
         uses: github/codeql-action/init@v1
       - name: Run vulnerability scanner (table)
         if: startsWith( github.ref, 'refs/tags/')
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "${{ steps.image_name.outputs.IMAGE_NAME }}:${{ steps.determine_tag.outputs.TAG_NAME }}"
+          image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
           format: "table"
       - name: Run vulnerability scanner (sarif)
         if: startsWith( github.ref, 'refs/tags/')
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "${{ steps.image_name.outputs.IMAGE_NAME }}:${{ steps.determine_tag.outputs.TAG_NAME }}"
+          image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
           format: "template"
           template: "@/contrib/sarif.tpl"
           output: "trivy-results.sarif"

.github/workflows/dockers-agent-sidecar-image.yml

@@ -67,13 +67,6 @@ jobs:
         uses: docker/setup-buildx-action@v1
         with:
           buildkitd-flags: "--debug"
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-vald-agent-sidecar-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-vald-agent-sidecar-
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
@@ -85,79 +78,26 @@ jobs:
           registry: ghcr.io
           username: ${{ secrets.PACKAGE_USER }}
           password: ${{ secrets.PACKAGE_TOKEN }}
-      - name: Image name
-        id: image_name
-        run: |
-          image_name=`make docker/name/agent-sidecar`
-          alter_org=`make docker/name/org/alter`
-          alter_image_name=`make ORG="${alter_org}" docker/name/agent-sidecar`
-          base_platforms=`make docker/platforms`
-          echo "IMAGE_NAME=${image_name}" >> $GITHUB_ENV
-          echo "ALTER_IMAGE_NAME=${alter_image_name}" >> $GITHUB_ENV
-          echo "::set-output name=IMAGE_NAME::${image_name}"
-          echo "::set-output name=BASE_PLATFORMS::${base_platforms}"
-      - name: Determine tag name (master)
-        if: github.ref == 'refs/heads/master'
-        run: |
-          echo "nightly" > versions/VALD_VERSION
-          echo "PRIMARY_TAG=nightly" >> $GITHUB_ENV
-          echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
-        env:
-          PLATFORMS: ${{ steps.image_name.outputs.BASE_PLATFORMS }}
-      - name: Determine tag name (pull request)
-        if: github.event_name == 'pull_request'
-        run: |
-          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
-          echo "PR-${pr_num}" > versions/VALD_VERSION
-          echo "PRIMARY_TAG=pr-${pr_num}" >> $GITHUB_ENV
-          echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
-        env:
-          PLATFORMS: linux/amd64
-      - name: Determine tag name (tags)
-        if: startsWith( github.ref, 'refs/tags/')
-        id: determine_tag
-        run: |
-          tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'`
-          echo "::set-output name=TAG_NAME::${tag_name}"
-          echo "PRIMARY_TAG=${tag_name}" >> $GITHUB_ENV
-          echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
-        env:
-          PLATFORMS: ${{ steps.image_name.outputs.BASE_PLATFORMS }}
-      - name: Add extra tags
-        run: |
-          EXTRA_TAGS="-t ${ALTER_IMAGE_NAME}:${PRIMARY_TAG}"
-          echo "EXTRA_TAGS=${EXTRA_TAGS}" >> $GITHUB_ENV
-      - name: Add latest tags
-        if: startsWith( github.ref, 'refs/tags/')
-        run: |
-          LATEST_TAGS="-t ${IMAGE_NAME}:latest -t ${ALTER_IMAGE_NAME}:latest"
-          echo "LATEST_TAGS=${LATEST_TAGS}" >> $GITHUB_ENV
-      - name: Build and Push
-        run: |
-          make \
-            DOCKER="docker buildx" \
-            DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${CACHE_OPTS} ${LABEL_OPTS} ${EXTRA_TAGS} ${LATEST_TAGS} --push" \
-            TAG="${PRIMARY_TAG}" \
-            docker/build/agent-sidecar
-        env:
-          DOCKER_BUILDKIT: 1
-          BUILDER: ${{ steps.buildx.outputs.name }}
-          CACHE_OPTS: "--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache"
-          LABEL_OPTS: "--label org.opencontainers.image.url=${{ github.event.repository.html_url }} --label org.opencontainers.image.source=${{ github.event.repository.html_url }} --label org.opencontainers.image.revision=${{ github.sha }}"
+      - name: Build and Publish
+        id: build_and_publish
+        uses: ./.github/actions/docker-build
+        with:
+          target: agent-sidecar
+          builder: ${{ steps.buildx.outputs.name }}
       - name: Initialize CodeQL
         if: startsWith( github.ref, 'refs/tags/')
         uses: github/codeql-action/init@v1
       - name: Run vulnerability scanner (table)
         if: startsWith( github.ref, 'refs/tags/')
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "${{ steps.image_name.outputs.IMAGE_NAME }}:${{ steps.determine_tag.outputs.TAG_NAME }}"
+          image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
           format: "table"
       - name: Run vulnerability scanner (sarif)
         if: startsWith( github.ref, 'refs/tags/')
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "${{ steps.image_name.outputs.IMAGE_NAME }}:${{ steps.determine_tag.outputs.TAG_NAME }}"
+          image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
           format: "template"
           template: "@/contrib/sarif.tpl"
           output: "trivy-results.sarif"