Two version deploy support (#2171)

* fix e2e test to work in a 2-version environment Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add workflow to create backport pr Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add reusable workflow to detect ci container image tag Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * use dynamic ci container image tag Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add workflow execution rule Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * deleted unnecessary workflow execution rule Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add new workflow to build docker images when release branch is created Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * refactor curl command Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * use dynamic ci container image for unit test and hack test Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add new workflow execution rule to codeql and protobuf build Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add workflow to release Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * make format Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * deleted unnecessary code Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add step to create release tag and release commit Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * deleted invalid environment variable Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add release logic and auto generate code for release Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * deleted unnecessary code Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * bugfix branch specification Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * bugfix release logic Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add checkout to release branch Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * bugfix set step result and add gpg import step Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix typo Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add new workflow for two version support Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add space to improve readability Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * deleted unnecessary line Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * bugfix github object error for release pr Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add cluster-info dump Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * change workflow name Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix: apply latest workflow changes and fix actionlint warning Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix: add new line for readability Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix: deleted unnecessary file Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * feat: add release branch image creation logic for new component Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix: execute make format to update license Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> --------- Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com>
vdaas · Jan 10, 2024 · 174969a · 174969a
1 parent 727c5ef
commit 174969a
Show file tree

Hide file tree

Showing 39 changed files with 864 additions and 254 deletions.
diff --git a/.github/actions/detect-docker-image-tags/action.yaml b/.github/actions/detect-docker-image-tags/action.yaml
@@ -52,6 +52,7 @@ runs:
           ["vdaas/vald-index-creation"]="manager.index.creator.image.tag"
           ["vdaas/vald-index-save"]="manager.index.saver.image.tag"
           ["vdaas/vald-helm-operator"]="image.tag"
+          ["vdaas/vald-ci-container"]=""
         )
 
         for image in ${IMAGES}

diff --git a/.github/actions/determine-docker-image-tag/action.yaml b/.github/actions/determine-docker-image-tag/action.yaml
@@ -20,6 +20,9 @@ outputs:
   PRIMARY_TAG:
     description: "Primary tag"
     value: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
+  DEFAULT_TAG:
+    description: "default tag. Tag to be used if the primary tag does not exist"
+    value: ${{ steps.determine_tag_name.outputs.DEFAULT_TAG }}
 
 runs:
   using: "composite"
@@ -28,6 +31,7 @@ runs:
       shell: bash
       run: |
         echo "GITHUB_REF $GITHUB_REF"
+        echo "GITHUB_BASE_REF $GITHUB_BASE_REF"
         echo "GITHUB_EVENT_PATH $GITHUB_EVENT_PATH"
         echo "GITHUB_EVENT_NAME ${{ github.event_name }}"
         echo "GITHUB_EVENT_NUMBER  ${{ github.event.number }}"
@@ -38,20 +42,40 @@ runs:
       run: |
         if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then
           tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'`
+          echo "${tag_name}" > versions/VALD_VERSION
+
           primary_tag="${tag_name}"
-        elif [ "${{ github.event_name }}" = "pull_request" ]; then
-          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
-          echo "PR-${pr_num}" > versions/VALD_VERSION
-          primary_tag="pr-${pr_num}"
-        elif [ "${{ github.event_name }}" = "pull_request_target" ]; then
+          default_tag="nightly"
+        elif [[ "${{ github.event_name }}" = "pull_request" || "${{ github.event_name }}" = "pull_request_target" ]]; then
           pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
           echo "PR-${pr_num}" > versions/VALD_VERSION
+
           primary_tag="pr-${pr_num}"
+          default_tag="nightly"
+
+          # For pull request to the release branch, use the release branch latest tag as the default tag (vx.x).
+          # This is only set if the event that triggers the workflow execution is pull_request or pull_request_target.
+          if [[ "$GITHUB_BASE_REF" =~ ^release/v([0-9]+)\.([0-9]+)$ ]]; then
+            tag_name=`echo $GITHUB_BASE_REF | sed -e 's:^release/::'`
+            default_tag="${tag_name}"
+          fi
         elif [ "$GITHUB_REF" = "refs/heads/main" ]; then
           echo "nightly" > versions/VALD_VERSION
+
           primary_tag="nightly"
+          default_tag="nightly"
+        elif [[ "$GITHUB_REF" =~ ^refs/heads/release/v([0-9]+)\.([0-9]+)$ ]]; then
+          tag_name=`echo $GITHUB_REF | sed -e 's:^refs/heads/release/::'`
+          echo "${tag_name}" > versions/VALD_VERSION
+
+          primary_tag="${tag_name}"
+          default_tag="nightly"
         else
           primary_tag="unknown"
+          default_tag="unknown"
         fi
+
         echo "PRIMARY_TAG is determined: ${primary_tag}"
+        echo "DEFAULT_TAG is determined: ${default_tag}"
         echo "PRIMARY_TAG=${primary_tag}" >> $GITHUB_OUTPUT
+        echo "DEFAULT_TAG=${default_tag}" >> $GITHUB_OUTPUT
diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml
@@ -15,6 +15,7 @@
 #
 name: "Build Docker images"
 description: "A action to build Docker images and publish them"
+
 inputs:
   target:
     description: "Build target"
@@ -44,6 +45,7 @@ outputs:
   EXTRA_TAGS:
     description: "Extra tags"
     value: ${{ steps.add_extra_tags.outputs.EXTRA_TAGS }}
+
 runs:
   using: "composite"
   steps:
@@ -60,9 +62,11 @@ runs:
         echo "ALTER_IMAGE_NAME=${alter_image_name}" >> $GITHUB_OUTPUT
       env:
         TARGET: ${{ inputs.target }}
+
     - name: Determine tag name
       id: determine_tag_name
       uses: ./.github/actions/determine-docker-image-tag
+
     - name: Determine platforms
       shell: bash
       id: determine_platforms
@@ -82,6 +86,7 @@ runs:
         echo "PLATFORMS=${platforms}" >> $GITHUB_OUTPUT
       env:
         TARGET_PLATFORMS: ${{ inputs.platforms }}
+
     - name: Add extra tags
       shell: bash
       id: add_extra_tags
@@ -97,6 +102,7 @@ runs:
         IMAGE_NAME: ${{ steps.image_name.outputs.IMAGE_NAME }}
         ALTER_IMAGE_NAME: ${{ steps.image_name.outputs.ALTER_IMAGE_NAME }}
         PRIMARY_TAG: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
+
     - name: Build and Push
       shell: bash
       id: build_and_push

diff --git a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml
@@ -41,6 +41,10 @@ inputs:
     description: "If you want to use local charts, set this to true."
     required: false
     default: "true"
+  default_image_tag:
+    description: "Default image tag. e.g) nightly, vx.x, vx.x.x, pr-xxx"
+    required: true
+    default: "nightly"
 outputs:
   POD_NAME:
     description: "A pod name that waited for"
@@ -67,22 +71,24 @@ runs:
       if: ${{ inputs.use_local_charts == 'false' }}
       run: |
         helm install vald-helm-operator \
-          --set image.tag=nightly \
+          --set image.tag=${DEFAULT_IMAGE_TAG} \
           ${HELM_EXTRA_OPTIONS} \
           charts/vald-helm-operator/.
 
         sleep 3
       env:
+        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
 
     - name: Deploy vald helm operator from local charts
       shell: bash
       id: deploy_vald_helm_operator_local
       if: ${{ inputs.use_local_charts == 'true' }}
       run: |
-        make k8s/vald-helm-operator/deploy VERSION=nightly HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
+        make k8s/vald-helm-operator/deploy VERSION=${DEFAULT_IMAGE_TAG} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
         sleep 3
       env:
+        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
 
     - name: Deploy vald

diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml
@@ -41,6 +41,10 @@ inputs:
     description: "If you want to use local charts, set this to true."
     required: false
     default: "true"
+  default_image_tag:
+    description: "Default image tag. e.g) nightly, vx.x, vx.x.x"
+    required: true
+    default: "nightly"
 outputs:
   POD_NAME:
     description: "A pod name that waited for"
@@ -68,7 +72,7 @@ runs:
       run: |
         helm install \
           --values ${VALUES} \
-          --set defaults.image.tag=nightly \
+          --set defaults.image.tag=${DEFAULT_IMAGE_TAG} \
           ${HELM_EXTRA_OPTIONS} \
           --generate-name charts/vald
 
@@ -81,6 +85,7 @@ runs:
         podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
         echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
       env:
+        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         VALUES: ${{ inputs.values }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
         WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }}
@@ -91,7 +96,7 @@ runs:
       id: deploy_vald_local
       if: ${{ inputs.use_local_charts == 'true' }}
       run: |
-        make k8s/vald/deploy VERSION=nightly HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
+        make k8s/vald/deploy VERSION=${DEFAULT_IMAGE_TAG} HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
 
         sleep 3
 
@@ -102,6 +107,7 @@ runs:
         podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
         echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
       env:
+        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         VALUES: ${{ inputs.values }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
         WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }}

diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml
@@ -15,6 +15,7 @@
 #
 name: "Setup E2E environment"
 description: "A action to set up the environment for executing E2E test"
+
 inputs:
   require_libhdf5:
     description: "If libhdf5 is not required, set this to false"
@@ -40,13 +41,18 @@ inputs:
     description: "Image names"
     required: false
     default: "vdaas/vald-agent-ngt vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-manager-index"
+
 outputs:
   HELM_EXTRA_OPTIONS:
     description: "Helm extra options that specifies E2E target image tags"
     value: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
   IMAGE_TAGS:
     description: "Specifies E2E target image tags"
     value: ${{ steps.specify_container_versions.outputs.IMAGE_TAGS }}
+  DEFAULT_IMAGE_TAG:
+    description: "Default E2E target image tag"
+    value: ${{ steps.determine_tag_name.outputs.DEFAULT_TAG }}
+
 runs:
   using: "composite"
   steps:

diff --git a/.github/workflows/_detect-ci-container.yml b/.github/workflows/_detect-ci-container.yml
@@ -0,0 +1,70 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Detect CI container image tag"
+on:
+  workflow_call:
+    outputs:
+      TAG_NAME:
+        description: "The docker image tag name"
+        value: ${{ jobs.detect.outputs.TAG_NAME }}
+
+env:
+  TARGET_IMAGE: vdaas/vald-ci-container
+
+jobs:
+  detect:
+    runs-on: ubuntu-latest
+    outputs:
+      TAG_NAME: ${{ steps.merge_detection_results.outputs.TAG_NAME }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Determine Docker image tag
+        id: determine_tag_name
+        uses: ./.github/actions/determine-docker-image-tag
+
+      - name: Detect Docker image tag for primary
+        id: detect_primary_tag_name
+        uses: ./.github/actions/detect-docker-image-tags
+        with:
+          images: ${{ env.TARGET_IMAGE }}
+          tag_name: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
+
+      - name: Detect Docker image tag for default
+        id: detect_default_tag_name
+        uses: ./.github/actions/detect-docker-image-tags
+        with:
+          images: ${{ env.TARGET_IMAGE }}
+          tag_name: ${{ steps.determine_tag_name.outputs.DEFAULT_TAG }}
+
+      - name: Merge Docker image tag detection results
+        id: merge_detection_results
+        run: |
+          TAG_NAME="nightly"
+
+          if [ -n "${DEFAULT_TAG_RESULT}" ]; then
+            TAG_NAME=${{ steps.determine_tag_name.outputs.DEFAULT_TAG }}
+          fi
+
+          if [ -n "${PRIMARY_TAG_RESULT}" ]; then
+            TAG_NAME=${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
+          fi
+
+          echo "TAG_NAME=${TAG_NAME}"
+          echo "TAG_NAME=${TAG_NAME}" >> $GITHUB_OUTPUT
+        env:
+          PRIMARY_TAG_RESULT: ${{ steps.detect_primary_tag_name.outputs.IMAGE_TAGS }}
+          DEFAULT_TAG_RESULT: ${{ steps.detect_default_tag_name.outputs.IMAGE_TAGS }}
diff --git a/.github/workflows/_docker-image-scan.yaml b/.github/workflows/_docker-image-scan.yaml
@@ -26,9 +26,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - name: Set Git config
         run: |
           git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
       - name: Build the Docker image
         id: build_image
         run: |
@@ -46,6 +48,7 @@ jobs:
           TARGET: ${{ inputs.target }}
           LABEL_OPTS: "--label org.opencontainers.image.url=${{ github.event.repository.html_url }} --label org.opencontainers.image.source=${{ github.event.repository.html_url }} --label org.opencontainers.image.revision=${{ github.sha }}"
           PRIMARY_TAG: ${{ github.sha }}
+
       - name: Scan the Docker image
         uses: ./.github/actions/scan-docker-image
         with: