Fix the logic to determine docker image (#2410) (#2420)

* fix: update action to get tag list



* fix: refactor sub command execution



* fix: use tag list in detect ci container workflow



* fix: name for detected result



* fix: invalid variable bug



* fix: execution error and invalid variable



* fix: deleted unnecessary comment



* fix: deleted unnecessary parameters



* fix: use new determine tag logic in the docker build workflow



* fix: bugfix extra tags handling and small refactor



* fix: build tag rule or release branch based on tag list



* fix: typo



* fix: indent



* fix: refactor syntax



* fix: typo and invalid option error



* fix: use helm extra option



* fix: add comment for tag management



* fix: unsetbug of helm extra option



* fix: build target condition



---------

Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com>
Co-authored-by: Hiroto Funakoshi <hiroto.funakoshi.hiroto@gmail.com>

.github/actions/detect-docker-image-tags/action.yaml

@@ -16,7 +16,7 @@
 name: "Detect Docker image tags"
 description: "A action to detect Docker image tags"
 inputs:
-  tag_name:
+  tags:
     description: "Tag name to check whether exists or not"
     required: true
     default: ""
@@ -53,15 +53,16 @@ runs:
           ["vdaas/vald-ci-container"]=""
         )
 
-        for image in ${IMAGES}
-        do
+        for image in ${IMAGES}; do
           echo "check for ${image}"
-
-          if curl -s "https://registry.hub.docker.com/v2/repositories/${image}/tags/${TAG_NAME}" | jq '.name' | grep -v "null"; then
-            echo "${image}:${TAG_NAME} exists. adding a helm option '--set ${m[${image}]}=${TAG_NAME}'."
-            export HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS} --set ${m[${image}]}=${TAG_NAME}"
-            export IMAGE_TAGS="${IMAGE_TAGS} ${m[${image}]}=\"${TAG_NAME}\""
-          fi
+          for tag in ${TAGS}; do
+            if curl -s "https://registry.hub.docker.com/v2/repositories/${image}/tags/${tag}" | jq '.name' | grep -v "null"; then
+              echo "${image}:${tag} exists. adding a helm option '--set ${m[${image}]}=${tag}'."
+              export HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS} --set ${m[${image}]}=${tag}"
+              export IMAGE_TAGS="${IMAGE_TAGS} ${m[${image}]}=\"${tag}\""
+              break
+            fi
+          done
         done
 
         echo "HELM_EXTRA_OPTIONS=${HELM_EXTRA_OPTIONS}"
@@ -70,4 +71,4 @@ runs:
         echo "IMAGE_TAGS=${IMAGE_TAGS}" >> $GITHUB_OUTPUT
       env:
         IMAGES: ${{ inputs.images }}
-        TAG_NAME: ${{ inputs.tag_name }}
+        TAGS: ${{ inputs.tags }}

.github/actions/determine-docker-image-tag/action.yaml

@@ -16,12 +16,12 @@
 name: "Determine Docker image tag"
 description: "A action to determine Docker image tag"
 outputs:
+  TAGS:
+    description: "docker image tag list"
+    value: ${{ steps.determine_tag_name.outputs.TAGS }}
   PRIMARY_TAG:
-    description: "Primary tag"
+    description: "highest priority docker image tag in tag list"
     value: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
-  DEFAULT_TAG:
-    description: "default tag. Tag to be used if the primary tag does not exist"
-    value: ${{ steps.determine_tag_name.outputs.DEFAULT_TAG }}
 runs:
   using: "composite"
   steps:
@@ -38,41 +38,63 @@ runs:
       id: determine_tag_name
       run: |
         if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then
-          tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'`
-          echo "${tag_name}" > versions/VALD_VERSION
+        # The following is the priority of tags:
+        #   tag -> release/vx.x tag (release branch build tag) -> commit hash tag -> vx.x -> latest tag
+        # When a tag is created, the image build for the tag may not be finished.
+        # In that case, the CI container will use the tag for the release branch.
+        # At the timing of the creation of the tag, the vx.x points to the previous version, so the priority is lowered.
+        # (At the timing of the start of the image build for tag, vx.x and tag are equal.)
+        tag=$(echo $GITHUB_REF | sed -e 's:^refs/tags/::') # e.g) v1.7.6
+        tags="${tag}"
 
-          primary_tag="${tag_name}"
-          default_tag="nightly"
-        elif [[ "${{ github.event_name }}" = "pull_request" || "${{ github.event_name }}" = "pull_request_target" ]]; then
-          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
-          echo "PR-${pr_num}" > versions/VALD_VERSION
+        release_branch_tag="release/${major_minor_tag}" # e.g) release/v1.7
+        tags="${tags} ${release_branch_tag}"
+
+        major_minor_tag="$(echo "${tag}" | sed -E 's/^v?([0-9]+\.[0-9]+).*$/v\1/')" # e.g) v.1.7
+        tags="${tags} ${major_minor_tag}"
 
-          primary_tag="pr-${pr_num}"
-          default_tag="nightly"
+        commit_hash_tag=${GITHUB_SHA::8}
+        tags="${tags} ${commit_hash_tag}"
+        tags="${tags} latest"
 
-          # For pull request to the release branch, use the release branch latest tag as the default tag (vx.x).
+        elif [[ "${{ github.event_name }}" = "pull_request" || "${{ github.event_name }}" = "pull_request_target" ]]; then
+          # The following is the priority of tags:
+          #   pr-xxx tag (PR build tag) -> release/vx.x tag (release branch tag) or nightly tag
+          pr_num=$(cat $GITHUB_EVENT_PATH | jq -r ".number")
+          tags="pr-${pr_num}"
+
+          # For pull request to the release branch, use the release branch latest tag as the default tag (release/vx.x).
           # This is only set if the event that triggers the workflow execution is pull_request or pull_request_target.
           if [[ "$GITHUB_BASE_REF" =~ ^release/v([0-9]+)\.([0-9]+)$ ]]; then
-            tag_name=`echo $GITHUB_BASE_REF | sed -e 's:^release/::'`
-            default_tag="${tag_name}"
+            tags="${tags} $(echo "$GITHUB_BASE_REF")"
+          else
+            # Currently, it is assumed to be nightly.
+            # However, further conditions will be added in the future if they cause inconvenience.
+            tags="${tags} nightly"
           fi
         elif [ "$GITHUB_REF" = "refs/heads/main" ]; then
-          echo "nightly" > versions/VALD_VERSION
+          # The following is the priority of tags:
+          #   commit hash tag -> nightly tag
+          commit_hash_tag=${GITHUB_SHA::8}
+          tags="${commit_hash_tag}"
+          tags="${tags} nightly"
 
-          primary_tag="nightly"
-          default_tag="nightly"
         elif [[ "$GITHUB_REF" =~ ^refs/heads/release/v([0-9]+)\.([0-9]+)$ ]]; then
-          tag_name=`echo $GITHUB_REF | sed -e 's:^refs/heads/release/::'`
-          echo "${tag_name}" > versions/VALD_VERSION
+        # The following is the priority of tags:
+        #   release/vx.x tag (release branch build tag) -> commit hash tag -> nightly tag
+        release_branch_tag=$(echo $GITHUB_REF | sed -e 's:^refs/heads/::') # e.g) release/v1.7
+        tags="${release_branch_tag}"
 
-          primary_tag="${tag_name}"
-          default_tag="nightly"
+        commit_hash_tag=${GITHUB_SHA::8}
+        tags="${tags} ${commit_hash_tag}"
+        tags="${tags} nightly"
         else
-          primary_tag="unknown"
-          default_tag="unknown"
+          tags="unknown"
         fi
+        primary_tag=$(echo ${tags} | awk '{print $1}')
+
+        echo "Determined tags: ${tags}"
+        echo "Primary tag: ${primary_tag}"
 
-        echo "PRIMARY_TAG is determined: ${primary_tag}"
-        echo "DEFAULT_TAG is determined: ${default_tag}"
+        echo "TAGS=${tags}"               >> $GITHUB_OUTPUT
         echo "PRIMARY_TAG=${primary_tag}" >> $GITHUB_OUTPUT
-        echo "DEFAULT_TAG=${default_tag}" >> $GITHUB_OUTPUT

.github/actions/docker-build/action.yaml

@@ -84,25 +84,32 @@ runs:
         echo "PLATFORMS=${platforms}" >> $GITHUB_OUTPUT
       env:
         TARGET_PLATFORMS: ${{ inputs.platforms }}
+        PRIMARY_TAG: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
+    - name: Update Vald version
+      shell: bash
+      run: echo "${PRIMARY_TAG}" >> versions/VALD_VERSION
+      env:
+        PRIMARY_TAG: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
     - name: Add extra tags
       shell: bash
       id: add_extra_tags
       run: |
-        extra_tags="-t ${ALTER_IMAGE_NAME}:${PRIMARY_TAG}"
-        if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then
-          latest_tags="-t ${IMAGE_NAME}:latest -t ${ALTER_IMAGE_NAME}:latest"
-          extra_tags="${extra_tags} ${latest_tags}"
-        fi
-        if [[ "$GITHUB_REF" =~ ^refs/heads/main$ ]] || [[ "$GITHUB_REF" =~ ^refs/heads/master$ ]] || [[ "$GITHUB_REF" =~ ^refs/heads/release.* ]] || [[ "${PRIMARY_TAG}" == "nightly" ]]; then
-          commit_hash=${GITHUB_SHA::8}
-          hash_tags="-t ${IMAGE_NAME}:${commit_hash} -t ${ALTER_IMAGE_NAME}:${commit_hash}"
-          extra_tags="${extra_tags} ${hash_tags}"
+        if [[ "${{ github.event_name }}" = "pull_request" || "${{ github.event_name }}" = "pull_request_target" ]]; then
+          extra_tags="-t ${IMAGE_NAME}:${PRIMARY_TAG} -t ${ALTER_IMAGE_NAME}:${PRIMARY_TAG}"
+        else
+          for tag in ${TAGS}; do
+            if [[ "${tag}" = "nightly" ]] && (! [[ "$GITHUB_REF" =~ ^refs/heads/main$ ]] || ! [[ "$GITHUB_REF" =~ ^refs/heads/master$ ]]); then
+              continue
+            fi
+            extra_tags="${extra_tags} -t ${IMAGE_NAME}:${tag} -t ${ALTER_IMAGE_NAME}:${tag}"
+          done
         fi
         echo "EXTRA_TAGS is determined: ${extra_tags}"
         echo "EXTRA_TAGS=${extra_tags}" >> $GITHUB_OUTPUT
       env:
         IMAGE_NAME: ${{ steps.image_name.outputs.IMAGE_NAME }}
         ALTER_IMAGE_NAME: ${{ steps.image_name.outputs.ALTER_IMAGE_NAME }}
+        TAGS: ${{ steps.determine_tag_name.outputs.TAGS }}
         PRIMARY_TAG: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
     - name: Build and Push
       shell: bash

.github/actions/e2e-deploy-vald-helm-operator/action.yaml

@@ -40,10 +40,6 @@ inputs:
     description: "If you want to use local charts, set this to true."
     required: false
     default: "true"
-  default_image_tag:
-    description: "Default image tag. e.g) nightly, vx.x, vx.x.x, pr-xxx"
-    required: true
-    default: "nightly"
 outputs:
   POD_NAME:
     description: "A pod name that waited for"
@@ -67,23 +63,20 @@ runs:
       if: ${{ inputs.use_local_charts == 'false' }}
       run: |
         helm install vald-helm-operator \
-          --set image.tag=${DEFAULT_IMAGE_TAG} \
           ${HELM_EXTRA_OPTIONS} \
           charts/vald-helm-operator/.
 
         sleep 6
       env:
-        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
     - name: Deploy vald helm operator from local charts
       shell: bash
       id: deploy_vald_helm_operator_local
       if: ${{ inputs.use_local_charts == 'true' }}
       run: |
-        make k8s/vald-helm-operator/deploy VERSION=${DEFAULT_IMAGE_TAG} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
+        make k8s/vald-helm-operator/deploy HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
         sleep 3
       env:
-        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
     - name: Deploy vald
       shell: bash

.github/actions/e2e-deploy-vald-readreplica/action.yaml

@@ -40,10 +40,6 @@ inputs:
     description: "If you want to use local charts, set this to true."
     required: false
     default: "true"
-  default_image_tag:
-    description: "Default image tag. e.g) nightly, vx.x, vx.x.x"
-    required: true
-    default: "nightly"
 outputs:
   POD_NAME:
     description: "A pod name that waited for"
@@ -68,7 +64,6 @@ runs:
       run: |
         helm install \
           --values ${VALUES} \
-          --set defaults.image.tag=${DEFAULT_IMAGE_TAG} \
           ${HELM_EXTRA_OPTIONS} \
           --generate-name charts/vald-readreplica
 
@@ -81,7 +76,6 @@ runs:
         podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
         echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
       env:
-        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         VALUES: ${{ inputs.values }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
         WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }}
@@ -91,7 +85,7 @@ runs:
       id: deploy_vald_readreplica_local
       if: ${{ inputs.use_local_charts == 'true' }}
       run: |
-        make k8s/vald-readreplica/deploy VERSION=${DEFAULT_IMAGE_TAG} HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
+        make k8s/vald-readreplica/deploy HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
 
         sleep 3
 
@@ -102,7 +96,6 @@ runs:
         podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
         echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
       env:
-        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         VALUES: ${{ inputs.values }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
         WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }}

.github/actions/e2e-deploy-vald/action.yaml

@@ -43,10 +43,6 @@ inputs:
     description: "If you want to use local charts, set this to true."
     required: false
     default: "true"
-  default_image_tag:
-    description: "Default image tag. e.g) nightly, vx.x, vx.x.x"
-    required: true
-    default: "nightly"
 outputs:
   POD_NAME:
     description: "A pod name that waited for"
@@ -78,7 +74,6 @@ runs:
       run: |
         helm install \
           --values ${VALUES} \
-          --set defaults.image.tag=${DEFAULT_IMAGE_TAG} \
           ${HELM_EXTRA_OPTIONS} \
           --generate-name charts/vald
 
@@ -91,7 +86,6 @@ runs:
         podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
         echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
       env:
-        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         VALUES: ${{ inputs.values }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
         WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }}
@@ -101,7 +95,7 @@ runs:
       id: deploy_vald_local
       if: ${{ inputs.use_local_charts == 'true' }}
       run: |
-        make k8s/vald/deploy VERSION=${DEFAULT_IMAGE_TAG} HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
+        make k8s/vald/deploy HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
 
         sleep 3
 
@@ -112,7 +106,6 @@ runs:
         podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
         echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
       env:
-        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         VALUES: ${{ inputs.values }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
         WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }}

.github/actions/setup-e2e/action.yaml

@@ -51,9 +51,6 @@ outputs:
   IMAGE_TAGS:
     description: "Specifies E2E target image tags"
     value: ${{ steps.specify_container_versions.outputs.IMAGE_TAGS }}
-  DEFAULT_IMAGE_TAG:
-    description: "Default E2E target image tag"
-    value: ${{ steps.determine_tag_name.outputs.DEFAULT_TAG }}
 runs:
   using: "composite"
   steps:
@@ -81,7 +78,7 @@ runs:
       id: specify_container_versions
       uses: ./.github/actions/detect-docker-image-tags
       with:
-        tag_name: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
+        tags: ${{ steps.determine_tag_name.outputs.TAGS }}
         images: ${{ inputs.target_images }}
     - uses: ./.github/actions/setup-k3d
       if: ${{ inputs.require_k3d == 'true' }}

.github/workflows/_detect-ci-container.yml

@@ -17,48 +17,32 @@ name: "Detect CI container image tag"
 on:
   workflow_call:
     outputs:
-      TAG_NAME:
+      TAG:
         description: "The docker image tag name"
-        value: ${{ jobs.detect.outputs.TAG_NAME }}
+        value: ${{ jobs.detect.outputs.TAG }}
 env:
   TARGET_IMAGE: vdaas/vald-ci-container
 jobs:
   detect:
     runs-on: ubuntu-latest
     outputs:
-      TAG_NAME: ${{ steps.merge_detection_results.outputs.TAG_NAME }}
+      TAG: ${{ steps.get_tag_name.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Determine Docker image tag
         id: determine_tag_name
         uses: ./.github/actions/determine-docker-image-tag
-      - name: Detect Docker image tag for primary
-        id: detect_primary_tag_name
+      - name: Detect Docker image tag
+        id: detect_tag_name
         uses: ./.github/actions/detect-docker-image-tags
         with:
           images: ${{ env.TARGET_IMAGE }}
-          tag_name: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
-      - name: Detect Docker image tag for default
-        id: detect_default_tag_name
-        uses: ./.github/actions/detect-docker-image-tags
-        with:
-          images: ${{ env.TARGET_IMAGE }}
-          tag_name: ${{ steps.determine_tag_name.outputs.DEFAULT_TAG }}
-      - name: Merge Docker image tag detection results
-        id: merge_detection_results
+          tags: ${{ steps.determine_tag_name.outputs.TAGS }}
+      - name: Get Docker image tag from detection result
+        id: get_tag_name
         run: |
-          TAG_NAME="nightly"
-
-          if [ -n "${DEFAULT_TAG_RESULT}" ]; then
-            TAG_NAME=${{ steps.determine_tag_name.outputs.DEFAULT_TAG }}
-          fi
-
-          if [ -n "${PRIMARY_TAG_RESULT}" ]; then
-            TAG_NAME=${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
-          fi
-
-          echo "TAG_NAME=${TAG_NAME}"
-          echo "TAG_NAME=${TAG_NAME}" >> $GITHUB_OUTPUT
+          TAG=$(echo "$TAGS" | awk '{print $1}' | awk -F '=' '{print $2}')
+          echo "TAG=${TAG}"
+          echo "TAG=${TAG}" >> $GITHUB_OUTPUT
         env:
-          PRIMARY_TAG_RESULT: ${{ steps.detect_primary_tag_name.outputs.IMAGE_TAGS }}
-          DEFAULT_TAG_RESULT: ${{ steps.detect_default_tag_name.outputs.IMAGE_TAGS }}
+          TAGS: ${{ steps.detect_tag_name.outputs.IMAGE_TAGS }}

.github/workflows/_release-pr.yml

@@ -40,7 +40,7 @@ jobs:
       - detect-ci-container
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }}
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     env:
       RELEASE_BRANCH_NAME: ${{ inputs.release_branch_name }}
       PREPARE_RELEASE_BRANCH_NAME: prepare/${{ inputs.release_branch_name }}