Merge branch 'main' into feature/protocol-buffer/migrate-to-buf

vdaas · Dec 7, 2023 · 297e72b · 297e72b
2 parents af1fcaf + 6eb3e08
commit 297e72b
Show file tree

Hide file tree

Showing 9 changed files with 1,082 additions and 925 deletions.
diff --git a/.github/actions/deploy-chaos-mesh/action.yaml b/.github/actions/deploy-chaos-mesh/action.yaml
@@ -36,15 +36,18 @@ runs:
         echo "version=${CHAOS_MESH_VERSION}" >> $GITHUB_OUTPUT
       env:
         CHAOS_MESH_VERSION: ${{ inputs.chaos_mesh_version }}
+
     - name: Check k3d
       shell: bash
       id: check_k3d
       run: |
         kubectl cluster-info dump
+
     - name: Check Helm
       shell: bash
       run: |
         helm version
+
     - name: Deploy Chaos Mesh
       shell: bash
       run: |

diff --git a/.github/actions/detect-docker-image-tags/action.yaml b/.github/actions/detect-docker-image-tags/action.yaml
@@ -49,6 +49,8 @@ runs:
           ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag"
           ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag"
           ["vdaas/vald-manager-index"]="manager.index.image.tag"
+          ["vdaas/vald-index-creation"]="manager.index.creator.image.tag"
+          ["vdaas/vald-index-save"]="manager.index.saver.image.tag"
           ["vdaas/vald-helm-operator"]="image.tag"
         )
 

diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml
@@ -56,31 +56,38 @@ runs:
       run: |
         sudo apt-get update
         sudo apt-get install -y libhdf5-dev
+
     - name: Setup Go environment
       if: ${{ inputs.require_go == 'true' }}
       uses: ./.github/actions/setup-go
+
     - name: Setup Helm environment
       if: ${{ inputs.require_helm == 'true' }}
       uses: ./.github/actions/setup-helm
+
     - name: Wait for target Docker images
       if: startsWith( github.ref, 'refs/tags/')
       uses: ./.github/actions/wait-for-docker-image
       with:
         images: ${{ inputs.target_images }}
+
     - name: Determine Docker image tag
       id: determine_tag_name
       uses: ./.github/actions/determine-docker-image-tag
+
     - name: Specify container versions
       id: specify_container_versions
       uses: ./.github/actions/detect-docker-image-tags
       with:
         tag_name: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
         images: ${{ inputs.target_images }}
+
     - uses: ./.github/actions/setup-k3d
       if: ${{ inputs.require_k3d == 'true' }}
       with:
         agents: 3
         ingress_port: ${{ inputs.ingress_port }}
+
     - name: Check Kubernetes cluster
       shell: bash
       run: |

diff --git a/.github/helm/values/values-index-management-jobs.yaml b/.github/helm/values/values-index-management-jobs.yaml
@@ -0,0 +1,31 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+agent:
+  ngt:
+    auto_index_duration_limit: 1000h
+    auto_index_check_duration: 1000h
+    auto_save_index_duration: 1000h
+
+manager:
+  index:
+    enabled: false
+    creator:
+      enabled: true
+      schedule: "* * * * *"
+    saver:
+      enabled: false
+      schedule: "2 * * * *"
diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml
@@ -219,9 +219,9 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - agent-failure
-      - random-pod-failure
       - agent-network-partition
       - clusterwide-network-bandwidth
+      - random-pod-failure
     steps:
       - uses: actions/checkout@v3
 

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -145,6 +145,57 @@ jobs:
         env:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
 
+  e2e-stream-crud-under-index-management-jobs:
+    name: "E2E test (Stream CRUD) under index management jobs"
+    needs: [dump-contexts-to-log]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set Git config
+        run: |
+          git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Setup E2E environment
+        id: setup_e2e
+        uses: ./.github/actions/setup-e2e
+        with:
+          target_images: "vdaas/vald-agent-ngt \
+            vdaas/vald-discoverer-k8s \
+            vdaas/vald-lb-gateway \
+            vdaas/vald-index-creation \
+            vdaas/vald-index-save"
+
+      - name: Deploy Vald
+        id: deploy_vald
+        uses: ./.github/actions/e2e-deploy-vald
+        env:
+          HELM_EXTRA_OPTIONS: "--values .github/helm/values/values-index-management-jobs.yaml"
+        with:
+          helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} ${{ env.HELM_EXTRA_OPTIONS }}
+          values: .github/helm/values/values-lb.yaml
+          wait_for_selector: app=vald-lb-gateway
+
+      - name: Run E2E CRUD
+        run: |
+          make hack/benchmark/assets/dataset/${{ env.DATASET }}
+          make E2E_BIND_PORT=8081 \
+            E2E_DATASET_NAME=${{ env.DATASET }} \
+            E2E_INSERT_COUNT=10000\
+            E2E_SEARCH_COUNT=10000 \
+            E2E_SEARCH_BY_ID_COUNT=10000 \
+            E2E_GET_OBJECT_COUNT=100 \
+            E2E_UPDATE_COUNT=100 \
+            E2E_UPSERT_COUNT=100 \
+            E2E_REMOVE_COUNT=100 \
+            E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
+            E2E_TARGET_POD_NAME=${POD_NAME} \
+            E2E_TARGET_NAMESPACE=default \
+            e2e
+        env:
+          POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
+
   e2e-stream-crud-skip-exist-check:
     name: "E2E test (Stream CRUD: skip strict exist check)"
     needs: [dump-contexts-to-log]
@@ -310,11 +361,12 @@ jobs:
     if: startsWith( github.ref, 'refs/tags/')
     needs:
       - e2e-agent-and-sidecar
+      - e2e-jobs
       - e2e-multiapis-crud
       - e2e-stream-crud
       - e2e-stream-crud-for-operator
       - e2e-stream-crud-skip-exist-check
-      - e2e-jobs
+      - e2e-stream-crud-under-index-management-jobs
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk
@@ -80,6 +80,8 @@ k8s/vald/deploy:
 	    --set gateway.filter.image.repository=$(CRORG)/$(FILTER_GATEWAY_IMAGE) \
 	    --set gateway.lb.image.repository=$(CRORG)/$(LB_GATEWAY_IMAGE) \
 	    --set manager.index.image.repository=$(CRORG)/$(MANAGER_INDEX_IMAGE) \
+	    --set manager.index.creator.image.repository=$(CRORG)/$(INDEX_CREATION_IMAGE) \
+	    --set manager.index.saver.image.repository=$(CRORG)/$(INDEX_SAVE_IMAGE) \
 	    $(HELM_EXTRA_OPTIONS) \
 	    --output-dir $(TEMP_DIR) \
 	    charts/vald
@@ -114,6 +116,8 @@ k8s/vald/delete:
 	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/save
 	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/creation
 	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/correction
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/creation
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/save
 	kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/lb
 	kubectl delete -f $(TEMP_DIR)/vald/templates/manager/index
 	kubectl delete -f $(TEMP_DIR)/vald/templates/discoverer

diff --git a/charts/vald-helm-operator/crds/valdrelease.yaml b/charts/vald-helm-operator/crds/valdrelease.yaml
@@ -388,6 +388,8 @@ spec:
                     readreplica:
                       type: object
                       properties:
+                        component_name:
+                          type: string
                         enabled:
                           type: boolean
                         label_key: